Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
01ca37ce
Commit
01ca37ce
authored
Nov 29, 2019
by
Markus Scheidgen
Browse files
Merge v0.6.4 and edit dialog with checkmarks and lifting embargo.
parent
350e43b2
Pipeline
#64654
passed with stages
in 13 minutes and 39 seconds
Changes
28
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
examples/external_project_parallel_upload/example-2.tar.gz
deleted
100644 → 0
View file @
350e43b2
File deleted
examples/external_project_parallel_upload/example-2.tar.gz
0 → 120000
View file @
01ca37ce
example-1.tar.gz
\ No newline at end of file
examples/external_project_parallel_upload/example-3.tar.gz
deleted
100644 → 0
View file @
350e43b2
File deleted
examples/external_project_parallel_upload/example-3.tar.gz
0 → 120000
View file @
01ca37ce
example-1.tar.gz
\ No newline at end of file
examples/external_project_parallel_upload/upload.py
View file @
01ca37ce
...
...
@@ -16,6 +16,7 @@ import tarfile
import
io
import
zipfile
import
zipstream
import
uuid
# config
nomad_url
=
'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/mp/api'
...
...
@@ -126,7 +127,7 @@ def upload_next_data(sources: Iterator[Tuple[str, str, str]], upload_name='next
yield
chunk
# stream .zip to nomad
response
=
requests
.
put
(
url
=
url
,
headers
=
{
'X-Token'
:
token
},
data
=
content
())
response
=
requests
.
put
(
url
=
url
,
headers
=
{
'X-Token'
:
token
,
'Content-type'
:
'application/octet-stream'
},
data
=
content
())
if
response
.
status_code
!=
200
:
raise
Exception
(
'nomad return status %d'
%
response
.
status_code
)
...
...
examples/files.py
0 → 100644
View file @
01ca37ce
from
nomad
import
infrastructure
,
files
,
processing
as
proc
infrastructure
.
setup_logging
()
infrastructure
.
setup_mongo
()
upload_id
=
'NvVyk3gATxCJW6dWS4cRWw'
upload
=
proc
.
Upload
.
get
(
upload_id
)
upload_with_metadata
=
upload
.
to_upload_with_metadata
()
upload_files
=
files
.
PublicUploadFiles
(
upload_id
)
upload_files
.
repack
(
upload_with_metadata
)
# try:
# public_upload_files = files.PublicUploadFiles(upload_id)
# public_upload_files.delete()
# except Exception:
# pass
# staging_upload_files = files.StagingUploadFiles(upload_id)
# staging_upload_files.pack(upload_with_metadata)
gui/src/components/ApiDialogButton.js
View file @
01ca37ce
...
...
@@ -56,10 +56,10 @@ class ApiDialogUnstyled extends React.Component {
}
<
/DialogContent
>
<
DialogActions
>
<
Button
onClick
=
{
this
.
handleToggleRaw
}
color
=
"
primary
"
>
<
Button
onClick
=
{
this
.
handleToggleRaw
}
>
{
showRaw
?
'
show tree
'
:
'
show raw JSON
'
}
<
/Button
>
<
Button
onClick
=
{
onClose
}
color
=
"
primary
"
>
<
Button
onClick
=
{
onClose
}
>
Close
<
/Button
>
<
/DialogActions
>
...
...
gui/src/components/EditUserMetadataDialog.js
View file @
01ca37ce
...
...
@@ -7,7 +7,7 @@ import DialogContent from '@material-ui/core/DialogContent'
import
DialogContentText
from
'
@material-ui/core/DialogContentText
'
import
DialogTitle
from
'
@material-ui/core/DialogTitle
'
import
PropTypes
from
'
prop-types
'
import
{
IconButton
,
Tooltip
,
withStyles
,
Paper
,
MenuItem
,
Popper
,
CircularProgress
}
from
'
@material-ui/core
'
import
{
IconButton
,
Tooltip
,
withStyles
,
Paper
,
MenuItem
,
Popper
,
CircularProgress
,
FormGroup
,
Checkbox
,
FormLabel
}
from
'
@material-ui/core
'
import
EditIcon
from
'
@material-ui/icons/Edit
'
import
AddIcon
from
'
@material-ui/icons/Add
'
import
RemoveIcon
from
'
@material-ui/icons/Delete
'
...
...
@@ -348,7 +348,7 @@ class DatasetInputUnstyled extends React.Component {
shouldRenderSuggestions
=
{()
=>
true
}
margin
=
{
margin
}
label
=
{
usedLabel
}
placeholder
=
{
`
Type the dataset's name
`
}
placeholder
=
"
Type the dataset's name
"
/>
}
}
...
...
@@ -398,6 +398,7 @@ class ReferenceInput extends React.Component {
onChange
=
{
this
.
handleChange
.
bind
(
this
)}
error
=
{
value
===
undefined
}
label
=
{
value
===
undefined
?
'
A reference must be a valid url
'
:
label
}
placeholder
=
"
Enter a URL to a related resource
"
/>
}
}
...
...
@@ -648,7 +649,7 @@ class InviteUserDialogUnstyled extends React.Component {
{
input
(
'
affiliation
'
,
'
Affiliation
'
)}
<
/DialogContent
>
<
DialogActions
>
<
Button
onClick
=
{
this
.
handleClose
.
bind
(
this
)}
color
=
"
primary
"
disabled
=
{
submitting
}
>
<
Button
onClick
=
{
this
.
handleClose
.
bind
(
this
)}
disabled
=
{
submitting
}
>
Cancel
<
/Button
>
<
div
className
=
{
classes
.
submitWrapper
}
>
...
...
@@ -665,6 +666,47 @@ class InviteUserDialogUnstyled extends React.Component {
const
InviteUserDialog
=
compose
(
withApi
(
true
,
false
),
withStyles
(
InviteUserDialogUnstyled
.
styles
))(
InviteUserDialogUnstyled
)
class
UserMetadataFieldUnstyled
extends
React
.
PureComponent
{
static
propTypes
=
{
classes
:
PropTypes
.
object
.
isRequired
,
children
:
PropTypes
.
node
,
modified
:
PropTypes
.
bool
,
onChange
:
PropTypes
.
func
.
isRequired
}
static
styles
=
theme
=>
({
root
:
{
flexWrap
:
'
nowrap
'
,
alignItems
:
'
flex-start
'
,
marginTop
:
theme
.
spacing
.
unit
*
2
},
container
:
{
width
:
'
100%
'
},
checkbox
:
{
marginLeft
:
-
theme
.
spacing
.
unit
*
2
,
marginRight
:
theme
.
spacing
.
unit
,
marginTop
:
theme
.
spacing
.
unit
}
})
render
()
{
const
{
children
,
classes
,
modified
,
onChange
}
=
this
.
props
return
<
FormGroup
row
className
=
{
classes
.
root
}
>
<
Checkbox
classes
=
{{
root
:
classes
.
checkbox
}}
checked
=
{
modified
}
onChange
=
{(
event
,
checked
)
=>
onChange
(
checked
)}
/
>
<
div
className
=
{
classes
.
container
}
>
{
children
}
<
/div
>
<
/FormGroup
>
}
}
const
UserMetadataField
=
withStyles
(
UserMetadataFieldUnstyled
.
styles
)(
UserMetadataFieldUnstyled
)
class
EditUserMetadataDialogUnstyled
extends
React
.
Component
{
static
propTypes
=
{
classes
:
PropTypes
.
object
.
isRequired
,
...
...
@@ -694,6 +736,9 @@ class EditUserMetadataDialogUnstyled extends React.Component {
left
:
'
50%
'
,
marginTop
:
-
12
,
marginLeft
:
-
12
},
liftEmbargoLabel
:
{
marginTop
:
theme
.
spacing
.
unit
*
3
}
})
...
...
@@ -710,7 +755,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
coauthors
:
[],
shared_with
:
[],
datasets
:
[],
with_embargo
:
true
with_embargo
:
'
lift
'
}
this
.
unmounted
=
false
}
...
...
@@ -720,13 +765,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
actions
:
{},
isVerifying
:
false
,
verified
:
true
,
submitting
:
false
,
testCoauthors
:
[],
testUser
:
{
message
:
null
,
success
:
true
,
value
:
null
}
submitting
:
false
}
componentWillUnmount
()
{
...
...
@@ -746,8 +785,7 @@ class EditUserMetadataDialogUnstyled extends React.Component {
shared_with
:
(
example
.
owners
||
[])
.
filter
(
user
=>
user
.
user_id
!==
example
.
uploader
.
user_id
)
.
map
(
user
=>
user
.
user_id
),
datasets
:
(
example
.
datasets
||
[]).
map
(
ds
=>
ds
.
name
),
with_embargo
:
example
.
with_embargo
datasets
:
(
example
.
datasets
||
[]).
map
(
ds
=>
ds
.
name
)
}
}
...
...
@@ -882,8 +920,16 @@ class EditUserMetadataDialogUnstyled extends React.Component {
const
dialogEnabled
=
user
&&
example
.
uploader
&&
example
.
uploader
.
user_id
===
user
.
sub
&&
!
disabled
const
submitEnabled
=
Object
.
keys
(
actions
).
length
&&
!
submitting
&&
verified
const
editDataToActions
=
editData
=>
{
if
(
Array
.
isArray
(
editData
))
{
return
editData
.
map
(
value
=>
({
value
:
value
}))
}
else
{
return
{
value
:
editData
}
}
}
const
listTextInputProps
=
(
key
,
verify
)
=>
{
const
values
=
actions
[
key
]
?
actions
[
key
]
:
this
.
editData
[
key
]
.
map
(
value
=>
({
value
:
value
})
)
const
values
=
actions
[
key
]
?
actions
[
key
]
:
editDataToActions
(
this
.
editData
[
key
])
return
{
values
:
values
,
...
...
@@ -897,6 +943,21 @@ class EditUserMetadataDialogUnstyled extends React.Component {
}
}
const
metadataFieldProps
=
(
key
,
verify
)
=>
({
modified
:
Boolean
(
actions
[
key
]),
onChange
:
checked
=>
{
if
(
checked
)
{
this
.
setState
({
actions
:
{...
actions
,
[
key
]:
editDataToActions
(
this
.
editData
[
key
])}},
()
=>
{
if
(
verify
)
{
this
.
verify
()
}
})
}
else
{
this
.
setState
({
actions
:
{...
actions
,
[
key
]:
undefined
}})
}
}
})
return
(
<
React
.
Fragment
>
<
IconButton
{...(
buttonProps
||
{})}
onClick
=
{
this
.
handleButtonClick
}
disabled
=
{
!
dialogEnabled
}
>
...
...
@@ -911,58 +972,63 @@ class EditUserMetadataDialogUnstyled extends React.Component {
<
DialogContentText
>
You
are
editing
{
total
}
{
total
===
1
?
'
entry
'
:
'
entries
'
}.
{
total
>
1
?
'
The fields are pre-filled with data from the first entry for.
'
:
''
}
Only
the
fields
that
you
change
will
be
updated
.
Be
aware
that
all
references
,
co
-
authors
,
shared
_
with
,
or
datasets
count
as
one
field
.
}
Only
the
checked
fields
will
be
updated
.
The
fields
references
,
co
-
authors
,
shared
with
users
,
and
datasets
can
have
many
values
.
Changing
one
value
,
will
apply
all
values
.
<
/DialogContentText
>
<
ActionInput
component
=
{
TextField
}
label
=
"
Comment
"
value
=
{
actions
.
comment
!==
undefined
?
actions
.
comment
:
{
value
:
this
.
editData
.
comment
}}
onChange
=
{
value
=>
this
.
setState
({
actions
:
{...
actions
,
comment
:
value
}})}
margin
=
"
normal
"
multiline
rows
=
"
4
"
fullWidth
placeholder
=
"
Add a comment
"
InputLabelProps
=
{{
shrink
:
true
}}
/
>
<
ListTextInput
component
=
{
ReferenceInput
}
{...
listTextInputProps
(
'
references
'
,
true
)}
label
=
"
References
"
/>
<
ListTextInput
component
=
{
UserInput
}
{...
listTextInputProps
(
'
coauthors
'
,
true
)}
label
=
"
Co-author
"
/>
<
ListTextInput
component
=
{
UserInput
}
{...
listTextInputProps
(
'
shared_with
'
,
true
)}
label
=
"
Shared with
"
/>
<
ListTextInput
component
=
{
DatasetInput
}
{...
listTextInputProps
(
'
datasets
'
,
true
)}
label
=
"
Datasets
"
/>
<
UserMetadataField
{...
metadataFieldProps
(
'
comment
'
)}
>
<
ActionInput
component
=
{
TextField
}
label
=
"
Comment
"
value
=
{
actions
.
comment
!==
undefined
?
actions
.
comment
:
{
value
:
this
.
editData
.
comment
}}
onChange
=
{
value
=>
this
.
setState
({
actions
:
{...
actions
,
comment
:
value
}})}
margin
=
"
normal
"
multiline
rowsMax
=
"
10
"
fullWidth
placeholder
=
"
Add a comment
"
InputLabelProps
=
{{
shrink
:
true
}}
/
>
<
/UserMetadataField
>
<
UserMetadataField
{...
metadataFieldProps
(
'
references
'
,
true
)}
>
<
ListTextInput
component
=
{
ReferenceInput
}
{...
listTextInputProps
(
'
references
'
,
true
)}
label
=
"
References
"
/>
<
/UserMetadataField
>
<
UserMetadataField
{...
metadataFieldProps
(
'
coauthors
'
,
true
)}
>
<
ListTextInput
component
=
{
UserInput
}
{...
listTextInputProps
(
'
coauthors
'
,
true
)}
label
=
"
Co-author
"
/>
<
/UserMetadataField
>
<
UserMetadataField
{...
metadataFieldProps
(
'
shared_with
'
,
true
)}
>
<
ListTextInput
component
=
{
UserInput
}
{...
listTextInputProps
(
'
shared_with
'
,
true
)}
label
=
"
Shared with
"
/>
<
/UserMetadataField
>
<
UserMetadataField
{...
metadataFieldProps
(
'
datasets
'
,
true
)}
>
<
ListTextInput
component
=
{
DatasetInput
}
{...
listTextInputProps
(
'
datasets
'
,
true
)}
label
=
"
Datasets
"
/>
<
/UserMetadataField
>
<
UserMetadataField
classes
=
{{
container
:
classes
.
liftEmbargoLabel
}}
{...
metadataFieldProps
(
'
with_embargo
'
,
true
)}
>
<
FormLabel
>
Lift
embargo
<
/FormLabel
>
<
/UserMetadataField
>
<
/DialogContent
>
{
Object
.
keys
(
actions
).
length
?
<
DialogContent
>
<
DialogContentText
>
The
following
fields
will
be
updated
with
the
given
values
:
<
i
>
{
Object
.
keys
(
actions
).
map
(
action
=>
action
).
join
(
'
,
'
)}
<
/i>
.
Updating
many
entries
might
take
a
few
seconds
.
<
/DialogContentText
>
<
/DialogContent
>
:
''
}
<
DialogActions
>
<
InviteUserDialog
/>
<
span
style
=
{{
flexGrow
:
1
}}
/
>
<
Button
onClick
=
{
this
.
handleClose
}
color
=
"
primary
"
disabled
=
{
submitting
}
>
<
Button
onClick
=
{
this
.
handleClose
}
disabled
=
{
submitting
}
>
Cancel
<
/Button
>
<
div
className
=
{
classes
.
submitWrapper
}
>
<
Button
onClick
=
{
this
.
handleSubmit
}
color
=
"
primary
"
disabled
=
{
!
submitEnabled
}
>
<
Button
onClick
=
{
this
.
handleSubmit
}
disabled
=
{
!
submitEnabled
}
color
=
"
primary
"
>
Submit
<
/Button
>
{
submitting
&&
<
CircularProgress
size
=
{
24
}
className
=
{
classes
.
submitProgress
}
/>
}
...
...
gui/src/components/entry/ArchiveEntryView.js
View file @
01ca37ce
...
...
@@ -66,7 +66,7 @@ class MetainfoDialogUnstyled extends React.PureComponent {
)}
data
=
{
metaInfoData
?
metaInfoData
.
miJson
:
{}}
title
=
"
Metainfo JSON
"
/>
<
Button
color
=
"
primary
"
onClick
=
{
onClose
}
>
<
Button
onClick
=
{
onClose
}
>
Close
<
/Button
>
<
/DialogActions
>
...
...
gui/src/components/entry/ArchiveLogView.js
View file @
01ca37ce
...
...
@@ -85,7 +85,7 @@ class ArchiveLogView extends React.Component {
<
Download
classes
=
{{
root
:
classes
.
downloadFab
}}
tooltip
=
"
download logfile
"
component
=
{
Fab
}
className
=
{
classes
.
downloadFab
}
color
=
"
prim
ary
"
size
=
"
medium
"
component
=
{
Fab
}
className
=
{
classes
.
downloadFab
}
color
=
"
second
ary
"
size
=
"
medium
"
url
=
{
`archive/logs/
${
uploadId
}
/
${
calcId
}
`
}
fileName
=
{
`
${
calcId
}
.log`
}
>
<
DownloadIcon
/>
...
...
gui/src/components/uploads/ConfirmDialog.js
View file @
01ca37ce
...
...
@@ -51,7 +51,7 @@ class ConfirmDialog extends React.Component {
<
/FormGroup
>
<
/DialogContent
>
<
DialogActions
>
<
Button
onClick
=
{
onClose
}
color
=
"
primary
"
>
<
Button
onClick
=
{
onClose
}
>
Cancel
<
/Button
>
<
Button
onClick
=
{()
=>
onPublish
(
withEmbargo
)}
color
=
"
primary
"
autoFocus
>
...
...
nomad/app/api/repo.py
View file @
01ca37ce
...
...
@@ -348,11 +348,15 @@ repo_edit_model = api.model('RepoEdit', {
})
def
edit
(
parsed_query
:
Dict
[
str
,
Any
],
logger
,
mongo_update
:
Dict
[
str
,
Any
]
=
None
,
re_index
=
True
):
def
edit
(
parsed_query
:
Dict
[
str
,
Any
],
logger
,
mongo_update
:
Dict
[
str
,
Any
]
=
None
,
re_index
=
True
)
->
List
[
str
]
:
# get all calculations that have to change
search_request
=
search
.
SearchRequest
()
add_query
(
search_request
,
parsed_query
)
calc_ids
=
list
(
hit
[
'calc_id'
]
for
hit
in
search_request
.
execute_scan
())
upload_ids
=
set
()
calc_ids
=
[]
for
hit
in
search_request
.
execute_scan
():
calc_ids
.
append
(
hit
[
'calc_id'
])
upload_ids
.
add
(
hit
[
'upload_id'
])
# perform the update on the mongo db
if
mongo_update
is
not
None
:
...
...
@@ -378,6 +382,8 @@ def edit(parsed_query: Dict[str, Any], logger, mongo_update: Dict[str, Any] = No
'edit repo with failed elastic updates'
,
payload
=
mongo_update
,
nfailed
=
len
(
failed
))
return
list
(
upload_ids
)
def
get_uploader_ids
(
query
):
""" Get all the uploader from the query, to check coauthers and shared_with for uploaders. """
...
...
@@ -427,6 +433,7 @@ class EditRepoCalcsResource(Resource):
# checking the edit actions and preparing a mongo update on the fly
mongo_update
=
{}
uploader_ids
=
None
lift_embargo
=
False
for
action_quantity_name
,
quantity_actions
in
actions
.
items
():
quantity
=
UserMetadata
.
m_def
.
all_quantities
.
get
(
action_quantity_name
)
if
quantity
is
None
:
...
...
@@ -437,9 +444,6 @@ class EditRepoCalcsResource(Resource):
if
not
g
.
user
.
is_admin
():
abort
(
404
,
'Only the admin user can set %s'
%
quantity
.
name
)
if
quantity
.
name
==
'Embargo'
:
abort
(
400
,
'Cannot raise an embargo, you can only lift the embargo'
)
if
isinstance
(
quantity_actions
,
list
)
==
quantity
.
is_scalar
:
abort
(
400
,
'Wrong shape for quantity %s'
%
action_quantity_name
)
...
...
@@ -491,7 +495,10 @@ class EditRepoCalcsResource(Resource):
name
=
action_value
)
dataset
.
m_x
(
'me'
).
create
()
mongo_value
=
dataset
.
dataset_id
elif
action_quantity_name
==
'with_embargo'
:
# ignore the actual value ... just lift the embargo
mongo_value
=
False
lift_embargo
=
True
else
:
mongo_value
=
action_value
...
...
@@ -519,7 +526,13 @@ class EditRepoCalcsResource(Resource):
return
json_data
,
400
# perform the change
edit
(
parsed_query
,
logger
,
mongo_update
,
True
)
upload_ids
=
edit
(
parsed_query
,
logger
,
mongo_update
,
True
)
# lift embargo
if
lift_embargo
:
for
upload_id
in
upload_ids
:
upload
=
proc
.
Upload
.
get
(
upload_id
)
upload
.
re_pack
()
return
json_data
,
200
...
...
@@ -643,7 +656,6 @@ class RepoQuantitiesResource(Resource):
quantities
=
args
.
get
(
'quantities'
,
[])
size
=
args
.
get
(
'size'
,
5
)
print
(
'A '
,
quantities
)
try
:
assert
size
>=
0
except
AssertionError
:
...
...
nomad/cli/admin/uploads.py
View file @
01ca37ce
...
...
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from
typing
import
List
from
typing
import
List
,
Callable
import
click
from
tabulate
import
tabulate
from
mongoengine
import
Q
...
...
@@ -204,11 +204,8 @@ def rm(ctx, uploads, skip_es, skip_mongo, skip_files):
upload
.
delete
()
@
uploads
.
command
(
help
=
'Reprocess selected uploads.'
)
@
click
.
argument
(
'UPLOADS'
,
nargs
=-
1
)
@
click
.
option
(
'--parallel'
,
default
=
1
,
type
=
int
,
help
=
'Use the given amount of parallel processes. Default is 1.'
)
@
click
.
pass_context
def
re_process
(
ctx
,
uploads
,
parallel
:
int
):
def
__run_processing
(
ctx
,
uploads
,
parallel
:
int
,
process
:
Callable
[[
proc
.
Upload
],
None
],
label
:
str
):
_
,
uploads
=
query_uploads
(
ctx
,
uploads
)
uploads_count
=
uploads
.
count
()
uploads
=
list
(
uploads
)
# copy the whole mongo query set to avoid cursor timeouts
...
...
@@ -223,29 +220,29 @@ def re_process(ctx, uploads, parallel: int):
logger
=
utils
.
get_logger
(
__name__
)
print
(
'%d uploads selected,
re-processing
...'
%
uploads_count
)
print
(
'%d uploads selected,
%s
...'
%
(
uploads_count
,
label
)
)
def
re_
process_upload
(
upload
:
proc
.
Upload
):
logger
.
info
(
'
re-processing
started'
,
upload_id
=
upload
.
upload_id
)
def
process_upload
(
upload
:
proc
.
Upload
):
logger
.
info
(
'
%s
started'
%
label
,
upload_id
=
upload
.
upload_id
)
completed
=
False
if
upload
.
process_running
:
logger
.
warn
(
'cannot trigger
re-proces
s, since the upload is already/still processing'
,
'cannot trigger
%
s, since the upload is already/still processing'
%
label
,
current_process
=
upload
.
current_process
,
current_task
=
upload
.
current_task
,
upload_id
=
upload
.
upload_id
)
else
:
upload
.
reset
()
upload
.
re_
process
_
upload
(
)
process
(
upload
)
upload
.
block_until_complete
(
interval
=
.
5
)
if
upload
.
tasks_status
==
proc
.
FAILURE
:
logger
.
info
(
'
re-processing
with failure'
,
upload_id
=
upload
.
upload_id
)
logger
.
info
(
'
%s
with failure'
%
label
,
upload_id
=
upload
.
upload_id
)
completed
=
True
logger
.
info
(
'
re-processing
complete'
,
upload_id
=
upload
.
upload_id
)
logger
.
info
(
'
%s
complete'
%
label
,
upload_id
=
upload
.
upload_id
)
with
cv
:
state
[
'completed_count'
]
+=
1
if
completed
else
0
...
...
@@ -253,8 +250,8 @@ def re_process(ctx, uploads, parallel: int):
state
[
'available_threads_count'
]
+=
1
print
(
'
re-processed
%s and skipped %s of %s uploads'
%
(
state
[
'completed_count'
],
state
[
'skipped_count'
],
uploads_count
))
'
%s
%s and skipped %s of %s uploads'
%
(
label
,
state
[
'completed_count'
],
state
[
'skipped_count'
],
uploads_count
))
cv
.
notify
()
...
...
@@ -262,7 +259,7 @@ def re_process(ctx, uploads, parallel: int):
with
cv
:
cv
.
wait_for
(
lambda
:
state
[
'available_threads_count'
]
>
0
)
state
[
'available_threads_count'
]
-=
1
thread
=
threading
.
Thread
(
target
=
lambda
:
re_
process_upload
(
upload
))
thread
=
threading
.
Thread
(
target
=
lambda
:
process_upload
(
upload
))
threads
.
append
(
thread
)
thread
.
start
()
...
...
@@ -270,6 +267,22 @@ def re_process(ctx, uploads, parallel: int):
thread
.
join
()
@
uploads
.
command
(
help
=
'Reprocess selected uploads.'
)
@
click
.
argument
(
'UPLOADS'
,
nargs
=-
1
)
@
click
.
option
(
'--parallel'
,
default
=
1
,
type
=
int
,
help
=
'Use the given amount of parallel processes. Default is 1.'
)
@
click
.
pass_context
def
re_process
(
ctx
,
uploads
,
parallel
:
int
):
__run_processing
(
ctx
,
uploads
,
parallel
,
lambda
upload
:
upload
.
re_process_upload
(),
're-processing'
)
@
uploads
.
command
(
help
=
'Repack selected uploads.'
)
@
click
.
argument
(
'UPLOADS'
,
nargs
=-
1
)
@
click
.
option
(
'--parallel'
,
default
=
1
,
type
=
int
,
help
=
'Use the given amount of parallel processes. Default is 1.'
)
@
click
.
pass_context
def
re_pack
(
ctx
,
uploads
,
parallel
:
int
):
__run_processing
(
ctx
,
uploads
,
parallel
,
lambda
upload
:
upload
.
re_pack
(),
're-packing'
)
@
uploads
.
command
(
help
=
'Attempt to abort the processing of uploads.'
)
@
click
.
argument
(
'UPLOADS'
,
nargs
=-
1
)
@
click
.
option
(
'--calcs'
,
is_flag
=
True
,
help
=
'Only stop calculation processing.'
)
...
...
nomad/files.py
View file @
01ca37ce
...
...
@@ -368,7 +368,9 @@ class StagingUploadFiles(UploadFiles):
def
archive_log_file_object
(
self
,
calc_id
:
str
)
->
PathObject
:
return
self
.
_archive_dir
.
join_file
(
'%s.log'
%
calc_id
)
def
add_rawfiles
(
self
,
path
:
str
,
move
:
bool
=
False
,
prefix
:
str
=
None
,
force_archive
:
bool
=
False
)
->
None
:
def
add_rawfiles
(
self
,
path
:
str
,
move
:
bool
=
False
,
prefix
:
str
=
None
,
force_archive
:
bool
=
False
,
target_dir
:
DirectoryObject
=
None
)
->
None
:
"""
Add rawfiles to the upload. The given file will be copied, moved, or extracted.
...
...
@@ -378,11 +380,12 @@ class StagingUploadFiles(UploadFiles):
prefix: Optional path prefix for the added files.
force_archive: Expect the file to be a zip or other support archive file.
Usually those files are only extracted if they can be extracted and copied instead.
target_dir: Overwrite the used directory to extract to. Default is the raw directory of this upload.
"""
assert
not
self
.
is_frozen
assert
os
.
path
.
exists
(
path
)
self
.
_size
+=
os
.
stat
(
path
).
st_size
target_dir
=
self
.
_raw_dir
target_dir
=
self
.
_raw_dir
if
target_dir
is
None
else
target_dir
if
prefix
is
not
None
:
target_dir
=
target_dir
.
join_dir
(
prefix
,
create
=
True
)
ext
=
os
.
path
.
splitext
(
path
)[
1
]
...
...
@@ -424,7 +427,7 @@ class StagingUploadFiles(UploadFiles):
def
pack
(
self
,
upload
:
UploadWithMetadata
,
target_dir
:
DirectoryObject
=
None
,
skip_raw
:
bool
=
False
)
->
None
:
skip_raw
:
bool
=
False
,
skip_archive
:
bool
=
False
)
->
None
:
"""
Replaces the staging upload data with a public upload record by packing all
data into files. It is only available if upload *is_bag*.
...
...
@@ -435,6 +438,7 @@ class StagingUploadFiles(UploadFiles):
target_dir: optional DirectoryObject to override where to put the files. Default
is the corresponding public upload files directory.
skip_raw: determine to not pack the raw data, only archive and user metadata
skip_raw: determine to not pack the archive data, only raw and user metadata
"""
self
.
logger
.
info
(
'started to pack upload'
)
...
...
@@ -464,6 +468,17 @@ class StagingUploadFiles(UploadFiles):
return
zipfile
.
ZipFile
(
file
.
os_path
,
mode
=
'w'
)
# zip archives
if
not
skip_archive
:
self
.
_pack_archive_files
(
upload
,
create_zipfile
)