Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
e1bcd545
Commit
e1bcd545
authored
Jan 30, 2020
by
Markus Scheidgen
Browse files
Sort upload grouped repo results by upload_time.
#247
parent
337fb2dc
Pipeline
#68306
passed with stages
in 14 minutes and 22 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
nomad/app/api/repo.py
View file @
e1bcd545
...
...
@@ -208,9 +208,13 @@ class RepoCalcsResource(Resource):
else
:
for
group_name
,
(
group_quantity
,
_
)
in
search
.
groups
.
items
():
if
args
.
get
(
group_name
,
False
):
kwargs
:
Dict
[
str
,
Any
]
=
{}
if
group_name
==
'uploads'
:
kwargs
.
update
(
order_by
=
'upload_time'
,
order
=
'desc'
)
search_request
.
quantity
(
group_quantity
,
size
=
per_page
,
examples
=
1
,
after
=
request
.
args
.
get
(
'%s_after'
%
group_name
,
None
))
after
=
request
.
args
.
get
(
'%s_after'
%
group_name
,
None
),
**
kwargs
)
results
=
search_request
.
execute_paginated
(
per_page
=
per_page
,
page
=
page
,
order
=
order
,
order_by
=
order_by
)
...
...
nomad/search.py
View file @
e1bcd545
...
...
@@ -477,7 +477,9 @@ class SearchRequest:
return
self
def
quantity
(
self
,
name
,
size
=
100
,
after
=
None
,
examples
=
0
,
examples_source
=
None
):
def
quantity
(
self
,
name
,
size
=
100
,
after
=
None
,
examples
=
0
,
examples_source
=
None
,
order_by
:
str
=
None
,
order
:
str
=
'desc'
):
"""
Adds a requests for values of the given quantity.
It allows to scroll through all values via elasticsearch's
...
...
@@ -501,6 +503,13 @@ class SearchRequest:
size:
The size gives the ammount of maximum values in the next scroll window.
If the size is None, a maximum of 100 quantity values will be requested.
examples:
Number of results to return that has each value
order_by:
A sortable quantity that should be used to order. The max of each
value bucket is used.
order:
"desc" or "asc"
"""
if
size
is
None
:
size
=
100
...
...
@@ -511,17 +520,25 @@ class SearchRequest:
# We are using elastic searchs 'composite aggregations' here. We do not really
# compose aggregations, but only those pseudo composites allow us to use the
# 'after' feature that allows to scan through all aggregation values.
composite
=
dict
(
sources
=
{
name
:
terms
},
size
=
size
)
if
order_by
is
None
:
composite
=
dict
(
sources
=
{
name
:
terms
},
size
=
size
)
else
:
sort_terms
=
A
(
'terms'
,
field
=
order_by
)
composite
=
dict
(
sources
=
[{
order_by
:
sort_terms
},
{
name
:
terms
}],
size
=
size
)
if
after
is
not
None
:
composite
[
'after'
]
=
{
name
:
after
}
if
order_by
is
None
:
composite
[
'after'
]
=
{
name
:
after
}
else
:
composite
[
'after'
]
=
{
order_by
:
after
,
name
:
''
}
composite_agg
=
self
.
_search
.
aggs
.
bucket
(
'quantity:%s'
%
name
,
'composite'
,
**
composite
)
composite
=
self
.
_search
.
aggs
.
bucket
(
'quantity:%s'
%
name
,
'composite'
,
**
composite
)
if
examples
>
0
:
kwargs
=
{}
kwargs
:
Dict
[
str
,
Any
]
=
{}
if
examples_source
is
not
None
:
kwargs
.
update
(
_source
=
dict
(
includes
=
examples_source
))
composite
.
metric
(
'examples'
,
A
(
'top_hits'
,
size
=
examples
,
**
kwargs
))
composite
_agg
.
metric
(
'examples'
,
A
(
'top_hits'
,
size
=
examples
,
**
kwargs
))
return
self
...
...
@@ -696,7 +713,14 @@ class SearchRequest:
result
=
dict
(
values
=
values
)
if
'after_key'
in
quantity
:
result
.
update
(
after
=
quantity
[
'after_key'
][
quantity_name
])
after
=
quantity
[
'after_key'
]
if
len
(
after
)
==
1
:
result
.
update
(
after
=
after
[
quantity_name
])
else
:
for
key
in
after
:
if
key
!=
quantity_name
:
result
.
update
(
after
=
after
[
key
])
break
return
result
...
...
tests/app/test_api.py
View file @
e1bcd545
...
...
@@ -790,7 +790,9 @@ class TestRepo():
uploads
=
data
.
get
(
'uploads'
,
None
)
assert
uploads
is
not
None
values
=
uploads
[
'values'
]
assert
values
[
'example_upload_id'
][
'total'
]
==
4
# the 4 uploads have "example upload id", but 3 have older upload time. Therefore,
# only 1 calc will be in the (today, "example upload id") bucket.
assert
values
[
'example_upload_id'
][
'total'
]
==
1
assert
values
[
'example_upload_id'
][
'examples'
][
0
][
'upload_id'
]
==
'example_upload_id'
assert
'after'
in
uploads
assert
'uploads'
in
data
[
'statistics'
][
'total'
][
'all'
]
...
...
tests/test_search.py
View file @
e1bcd545
...
...
@@ -137,7 +137,11 @@ def test_search_totals(elastic, example_search_data):
assert
'quantities'
not
in
results
def
test_search_quantity
(
elastic
,
normalized
:
parsing
.
LocalBackend
,
test_user
:
datamodel
.
User
,
other_test_user
:
datamodel
.
User
):
@
pytest
.
mark
.
parametrize
(
"order_by"
,
[
None
,
'upload_id'
])
def
test_search_quantity
(
elastic
,
normalized
:
parsing
.
LocalBackend
,
test_user
:
datamodel
.
User
,
other_test_user
:
datamodel
.
User
,
order_by
:
str
):
calc_with_metadata
=
datamodel
.
CalcWithMetadata
(
upload_id
=
'test upload id'
,
calc_id
=
'test id'
)
calc_with_metadata
.
apply_domain_metadata
(
normalized
)
calc_with_metadata
.
uploader
=
test_user
.
user_id
...
...
@@ -148,12 +152,17 @@ def test_search_quantity(elastic, normalized: parsing.LocalBackend, test_user: d
create_entry
(
calc_with_metadata
)
refresh_index
()
request
=
SearchRequest
().
quantity
(
name
=
'authors'
,
size
=
1
,
examples
=
1
)
request
=
SearchRequest
().
quantity
(
name
=
'authors'
,
size
=
1
,
examples
=
1
,
order_by
=
order_by
)
results
=
request
.
execute
()
assert
len
(
results
[
'quantities'
][
'authors'
][
'values'
].
keys
())
==
1
name
=
list
(
results
[
'quantities'
][
'authors'
][
'values'
].
keys
())[
0
]
assert
len
(
results
[
'quantities'
][
'authors'
][
'values'
][
name
][
'examples'
])
==
1
assert
results
[
'quantities'
][
'authors'
][
'after'
]
==
name
if
order_by
is
None
:
assert
results
[
'quantities'
][
'authors'
][
'after'
]
==
name
else
:
assert
results
[
'quantities'
][
'authors'
][
'after'
]
==
\
results
[
'quantities'
][
'authors'
][
'values'
][
name
][
'examples'
][
0
][
order_by
]
def
refresh_index
():
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment