Commit b2ea57d0 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Work in progress, added new raw file listing API, started to adapt raw file GUI. [skip-ci]

parent f834c633
Pipeline #52089 skipped
......@@ -9,7 +9,8 @@ import Download from './Download'
class RawFiles extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
data: PropTypes.object.isRequired,
uploadId: PropTypes.str.isRequired,
mainfile: PropTypes.str.isRequired,
api: PropTypes.object.isRequired,
user: PropTypes.object,
loading: PropTypes.number.isRequired
......@@ -23,7 +24,28 @@ class RawFiles extends React.Component {
})
state = {
selectedFiles: []
selectedFiles: [],
files: null
}
componentDidMount() {
this.update()
}
componentDidUpdate(prevProps) {
if (prevProps.api !== this.props.api) {
this.update()
}
}
update() {
const {uploadId, mainfile} = this.props
this.props.api.raw_file_list(uploadId, mainfile).then(data => {
this.setState({files: data.files})
}).catch(error => {
this.setState({files: null})
this.props.raiseError(error)
})
}
label(file) {
......@@ -42,9 +64,22 @@ class RawFiles extends React.Component {
}
render() {
const {classes, data: {files, upload_id, calc_id}, loading} = this.props
const availableFiles = files || []
const {selectedFiles} = this.state
const {classes, uploadId, mainfile, loading} = this.props
const {selectedFiles, files} = this.state
const mainfileLocal = mainfile.split('/')[-1]
let availableFiles = [
{
file: mainfileLocal,
size: -1
}
]
if (files) {
const mainfileIndex = files.findIndex(file => file.file === mainfileLocal)
}
const someSelected = selectedFiles.length > 0
const allSelected = availableFiles.length === selectedFiles.length && someSelected
......@@ -65,7 +100,7 @@ class RawFiles extends React.Component {
</FormLabel>
<Download component={IconButton} disabled={selectedFiles.length === 0}
tooltip="download selected files"
url={(selectedFiles.length === 1) ? `raw/${upload_id}/${selectedFiles[0]}` : `raw/${calc_id}?files=${encodeURIComponent(selectedFiles.join(','))}`}
url={(selectedFiles.length === 1) ? `raw/${uploadId}/${selectedFiles[0]}` : `raw/${calc_id}?files=${encodeURIComponent(selectedFiles.join(','))}`}
fileName={selectedFiles.length === 1 ? this.label(selectedFiles[0]) : `${calc_id}.zip`}
>
<DownloadIcon />
......
......@@ -31,6 +31,14 @@ from .auth import login_if_available, create_authorization_predicate, \
ns = api.namespace('raw', description='Downloading raw data files.')
raw_file_list_model = api.model('RawFileList', {
'upload_id': fields.String(description='The id of the upload.'),
'directory': fields.String(description='The path to the directory in the upload.'),
'contents': fields.List(
fields.Nested(model=api.model('RawFileListContents', {
'file': fields.String(description='The file name'),
'size': fields.Integer(description='The file size in bytes')
})))})
raw_file_compress_argument = dict(
name='compress', type=bool, help='Use compression on .zip files, default is not.',
......@@ -40,6 +48,43 @@ raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
raw_file_from_path_parser.add_argument(**signature_token_argument)
@ns.route('/list/<string:upload_id>/<path:directory>')
@api.doc(params={
'upload_id': 'The unique id for the requested upload.',
'directory': 'The directory in the upload with the desired contents.'
})
@api.header('Content-Type', 'application/json')
class RawFileList(Resource):
@api.doc('get')
@api.response(404, 'The upload or path does not exist')
@api.response(401, 'Not authorized to access the data.')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/json'})
@api.marshal_with(raw_file_list_model, skip_none=True, code=200, description='File list send')
@login_if_available
@with_signature_token
def get(self, upload_id: str, directory: str):
"""
Get the contents of the given directory for the given upload.
If the path points to a file a single entry is returned. If the path
points to a directory, information on all files in the directory are returned.
"""
upload_files = UploadFiles.get(upload_id, create_authorization_predicate(upload_id))
if upload_files is None:
abort(404, message='The upload with id %s does not exist.' % upload_id)
files = upload_files.raw_file_list(directory=directory)
if len(files) == 0:
abort(404, message='There are no files for %s.' % directory)
else:
return {
'upload_id': upload_id,
'directory': directory,
'contents': [dict(file=file, size=size) for file, size in files]
}
@ns.route('/<string:upload_id>/<path:path>')
@api.doc(params={
'upload_id': 'The unique id for the requested upload.',
......
......@@ -49,7 +49,7 @@ being other mainfiles. Therefore, the aux files of a restricted calc might becom
"""
from abc import ABCMeta
from typing import IO, Generator, Dict, Iterable, Callable
from typing import IO, Generator, Dict, Iterable, Callable, List, Tuple
import os.path
import os
import shutil
......@@ -259,6 +259,16 @@ class UploadFiles(DirectoryObject, metaclass=ABCMeta):
"""
raise NotImplementedError()
def raw_file_list(self, directory: str) -> List[Tuple[str, int]]:
"""
Gives a list of directory contents and its size.
Arguments:
directory: The directory to list
Returns:
A list of tuples with file name and size.
"""
raise NotImplementedError()
def archive_file(self, calc_id: str, *args, **kwargs) -> IO:
"""
Opens a archive file and returns a file-like objects. Additional args, kwargs are
......@@ -502,6 +512,23 @@ class StagingUploadFiles(UploadFiles):
if path_prefix is None or path.startswith(path_prefix):
yield path
def raw_file_list(self, directory: str) -> List[Tuple[str, int]]:
if directory is None or directory == '':
prefix = self._raw_dir.os_path
else:
prefix = os.path.join(self._raw_dir.os_path, directory)
results: List[Tuple[str, int]] = []
if not os.path.isdir(prefix):
return results
for file in os.listdir(prefix):
path = os.path.join(prefix, file)
if os.path.isfile(path):
results.append((file, os.path.getsize(path)))
return results
def calc_files(self, mainfile: str, with_mainfile: bool = True) -> Iterable[str]:
"""
Returns all the auxfiles and mainfile for a given mainfile. This implements
......@@ -677,6 +704,30 @@ class PublicUploadFiles(UploadFiles):
except FileNotFoundError:
pass
def raw_file_list(self, directory: str) -> List[Tuple[str, int]]:
if directory is None:
directory = ''
directory_len = len(directory)
results = []
for access in ['public', 'restricted']:
try:
zf = self.get_zip_file('raw', access, 'plain')
for path in zf.namelist():
content_path = path[directory_len + (0 if directory_len == 0 else 1):]
if path.startswith(directory) and '/' not in content_path:
if '/' not in content_path:
results.append((content_path, zf.getinfo(path).file_size))
else:
# this asserts that sub directories are always behind their
# parents and file siblings
break
except FileNotFoundError:
pass
return results
def archive_file(self, calc_id: str, *args, **kwargs) -> IO:
return self._file('archive', self._archive_ext, '%s.%s' % (calc_id, self._archive_ext), *args, **kwargs)
......
class MObject:
def __init__(self, m_definition: 'MElementDef', m_section: 'MSection' = None):
self.m_definition = m_definition
self.m_section = m_section
@property
def m_section(self):
return self._section
@m_section.setter
def m_section(self, m_section: 'MSection'):
self._section = m_section
# add yourself to the parent section
if m_section is not None:
subsection = m_section.subsections.get(self.m_definition.name)
if subsection is None:
subsection = []
m_section.subsections[self.m_definition.name] = subsection
subsection.append(self)
class MSection(MObject):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.subsections = dict()
self.properties = dict()
def __getattr__(self, name: str):
if name.startswith('new_'):
name = name[len('new_'):]
subsection = self.m_definition.get_subsection(name)
if subsection is None:
raise KeyError('Section "%s" does not have subsection "%s", available subsections are %s' % (self.m_definition.name, name, '?'))
def constructor(**kwargs):
new_section = subsection.impl(m_definition=subsection, m_section=self)
for key, value in kwargs.items():
setattr(new_section, key, value)
return new_section
return constructor
def __setattr__(self, name: str, value):
try:
super().__setattr__(name, value)
except KeyError:
self.__dict__[name] = value
def __repr__(self):
return ':%s' % self.m_definition.name
class MProperty(MObject):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.value = None
class MElementDef(MSection):
def __init__(self, m_definition: 'MElementDef', name: str): # more **kwargs
super().__init__(m_definition=m_definition)
self.name = name
def __repr__(self):
return '%s:%s' % (self.name, self.m_definition.name)
class MSectionDef(MElementDef):
def __init__(self, abstract: bool = False, repeats: bool = False, impl=MSection, **kwargs):
super().__init__(**kwargs)
self.repeats = repeats
self.abstract = abstract
self.impl = impl
def get_subsection(self, name: str):
return self.m_section.get_section(name)
m_section = MSectionDef(m_definition=None, name='section', repeats=True, impl=MSectionDef)
m_section.m_definition = m_section
m_element = MSectionDef(m_definition=m_section, name='element', abstract=True, impl=None)
m_package = MSectionDef(m_definition=m_section, name='package')
class MPackageDef(MElementDef):
def __init__(self, name: str):
super().__init__(m_definition=m_package, name=name)
def get_section(self, name: str):
# TODO add cache
for section in self.subsections['section']:
if section.name == name:
return section
return None
metainfo = MPackageDef(name='metainfo')
m_element.m_section = metainfo
m_section.m_section = metainfo
m_package.m_section = metainfo
m_property = metainfo.new_section(name='property', repeats=True, extends=m_element, section=m_package)
print(metainfo.subsections['section'])
metainfo.new_property(name='abstract', type=bool, section=m_element)
......@@ -863,7 +863,7 @@ class TestRaw(UploadFilesBasedTests):
assert 'files' not in data
@UploadFilesBasedTests.ignore_authorization
def test_raw_file_listing(self, client, upload, auth_headers):
def test_raw_file_list_alternatives(self, client, upload, auth_headers):
url = '/raw/%s/examples' % upload
rv = client.get(url, headers=auth_headers)
assert rv.status_code == 404
......@@ -959,6 +959,26 @@ class TestRaw(UploadFilesBasedTests):
assert rv.status_code == 404
@UploadFilesBasedTests.ignore_authorization
def test_raw_files_list(self, client, upload, auth_headers):
url = '/raw/list/%s/examples_template' % upload
rv = client.get(url, headers=auth_headers)
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['contents']) == 5
assert data['upload_id'] == upload
assert data['directory'] == 'examples_template'
for content in data['contents']:
assert content['file'] is not None
assert content['size'] >= 0
@UploadFilesBasedTests.ignore_authorization
def test_raw_files_list_missing(self, client, upload, auth_headers):
url = '/raw/list/%s/examples_' % upload
rv = client.get(url, headers=auth_headers)
assert rv.status_code == 404
def test_docs(client):
rv = client.get('/docs/index.html')
......
......@@ -221,6 +221,21 @@ class UploadFilesContract(UploadFilesFixtures):
raw_files = list(upload_files.raw_file_manifest(path_prefix=prefix))
assert_example_files(raw_files)
@pytest.mark.parametrize('prefix', [None, 'examples_template'])
def test_raw_file_list(self, test_upload: UploadWithFiles, prefix: str):
_, upload_files = test_upload
raw_files = list(upload_files.raw_file_list(directory=prefix))
if prefix is None:
assert len(raw_files) == 0
else:
assert '1.aux' in list(path for path, _ in raw_files)
for file, size in raw_files:
if file.endswith('.aux'):
assert size == 0
else:
assert size > 0
assert_example_files([os.path.join(prefix, path) for path, _ in raw_files])
@pytest.mark.parametrize('test_logs', [True, False])
def test_archive(self, test_upload: UploadWithFiles, test_logs: bool):
upload, upload_files = test_upload
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment