Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
5baa6b69
Commit
5baa6b69
authored
Jan 20, 2020
by
Markus Scheidgen
Browse files
Added non direct streaming upload version to the upload example.
parent
61f6b012
Pipeline
#67288
passed with stages
in 27 minutes and 39 seconds
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
examples/external_project_parallel_upload/upload.py
View file @
5baa6b69
...
...
@@ -24,6 +24,7 @@ user = 'leonard.hofstadter@nomad-fairdi.tests.de'
password
=
'password'
approx_upload_size
=
32
*
1024
*
1024
*
1024
# you can make it really small for testing
max_parallel_uploads
=
9
direct_stream
=
False
# create the bravado client
host
=
urlparse
(
nomad_url
).
netloc
.
split
(
':'
)[
0
]
...
...
@@ -115,8 +116,6 @@ def upload_next_data(sources: Iterator[Tuple[str, str, str]], upload_name='next
zip_stream
=
zipstream
.
ZipFile
(
mode
=
'w'
,
compression
=
zipfile
.
ZIP_STORED
,
allowZip64
=
True
)
zip_stream
.
paths_to_write
=
iterator
()
zip_stream
user
=
client
.
auth
.
get_user
().
response
().
result
token
=
user
.
token
url
=
nomad_url
+
'/uploads/?%s'
%
urlencode
(
dict
(
name
=
upload_name
))
...
...
@@ -126,8 +125,18 @@ def upload_next_data(sources: Iterator[Tuple[str, str, str]], upload_name='next
if
len
(
chunk
)
!=
0
:
yield
chunk
# stream .zip to nomad
response
=
requests
.
put
(
url
=
url
,
headers
=
{
'X-Token'
:
token
,
'Content-type'
:
'application/octet-stream'
},
data
=
content
())
if
direct_stream
:
# stream .zip to nomad
response
=
requests
.
put
(
url
=
url
,
headers
=
{
'X-Token'
:
token
,
'Content-type'
:
'application/octet-stream'
},
data
=
content
())
else
:
# save .zip and upload file to nomad
zipfile_name
=
'/tmp/%s.zip'
%
str
(
uuid
.
uuid4
())
with
open
(
zipfile_name
,
'wb'
)
as
f
:
for
c
in
content
():
f
.
write
(
c
)
with
open
(
zipfile_name
,
'rb'
)
as
f
:
response
=
requests
.
put
(
url
=
url
,
headers
=
{
'X-Token'
:
token
,
'Content-type'
:
'application/octet-stream'
},
data
=
f
)
if
response
.
status_code
!=
200
:
raise
Exception
(
'nomad return status %d'
%
response
.
status_code
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment