Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
N
nomad-lab-base
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
This is an archived project. Repository and other project resources are read-only.
Show more breadcrumbs
nomad-lab
nomad-lab-base
Commits
9944487c
Commit
9944487c
authored
6 years ago
by
Ihrig, Arvid Conrad (ari)
Browse files
Options
Downloads
Patches
Plain Diff
Integrated Pipeline: CalculationParsingFlow test refactoring
parent
d8cddf46
Branches
Branches containing commit
Tags
Tags containing commit
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/CalculationParsingFlowSpec.scala
+25
-35
25 additions, 35 deletions
...ntegrated_pipeline_tests/CalculationParsingFlowSpec.scala
with
25 additions
and
35 deletions
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/CalculationParsingFlowSpec.scala
+
25
−
35
View file @
9944487c
...
@@ -29,34 +29,25 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -29,34 +29,25 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
relativePath
=
Paths
.
get
(
fileName
),
relativePath
=
Paths
.
get
(
fileName
),
parserName
=
"dummyParser"
parserName
=
"dummyParser"
)
)
val
unusedPath
=
Paths
.
get
(
"/tmp/extracted/magic"
)
private
class
GraphWithDummyWorkers
(
numWorkers
:
Int
)
extends
StreamAssertions
[
FileParsingResultSignal
]
{
private
class
Fixture
(
numWorkers
:
Int
)
extends
StreamAssertions
[
FileParsingResultSignal
]
{
require
(
numWorkers
>=
1
,
"need at least one dummy worker"
)
require
(
numWorkers
>=
1
,
"need at least one dummy worker"
)
val
dummyWorkers
=
(
1
to
numWorkers
).
map
{
index
=>
val
dummyWorkers
=
(
1
to
numWorkers
).
map
{
index
=>
val
myMock
=
mock
[
CalculationParsingEngine
]
val
myMock
=
mock
[
CalculationParsingEngine
]
when
(
myMock
.
processRequest
(
any
(),
any
())).
thenAnswer
(
new
Answer
[
FileParsingResult
]
{
when
(
myMock
.
processRequest
(
any
(),
any
())).
thenAnswer
(
new
Answer
[
FileParsingResult
]
{
override
def
answer
(
invocation
:
InvocationOnMock
)
:
FileParsingResult
=
{
override
def
answer
(
invocation
:
InvocationOnMock
)
:
FileParsingResult
=
{
val
task
=
invocation
.
getArgument
[
FileParsingTask
](
0
)
anInMemoryResult
().
InMemoryResult
(
withTask
(
invocation
.
getArgument
[
FileParsingTask
](
0
)).
task
=
task
,
withParseResult
(
ParseResult
.
ParseSuccess
).
result
=
ParseResult
.
ParseSuccess
,
build
()
start
=
None
,
events
=
Seq
(
AddValue
(
"dummy_worker_index"
,
JInt
(
index
))),
end
=
None
)
}
}
})
})
when
(
myMock
.
failParseRequest
(
any
(),
any
())).
thenAnswer
(
new
Answer
[
FileParsingResult
]
{
when
(
myMock
.
failParseRequest
(
any
(),
any
())).
thenAnswer
(
new
Answer
[
FileParsingResult
]
{
override
def
answer
(
invocation
:
InvocationOnMock
)
:
FileParsingResult
=
{
override
def
answer
(
invocation
:
InvocationOnMock
)
:
FileParsingResult
=
{
val
task
=
invocation
.
getArgument
[
FileParsingTask
](
0
)
anInMemoryResult
().
InMemoryResult
(
withTask
(
invocation
.
getArgument
[
FileParsingTask
](
0
)).
task
=
task
,
withParseResult
(
ParseResult
.
ParseFailure
).
result
=
ParseResult
.
ParseFailure
,
build
()
start
=
None
,
events
=
Seq
(
AddValue
(
"dummy_worker_index"
,
JInt
(
index
))),
end
=
None
)
}
}
})
})
myMock
myMock
...
@@ -65,7 +56,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -65,7 +56,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
private
val
testInput
=
TestSource
.
probe
[
FileParsingTaskSignal
]
private
val
testInput
=
TestSource
.
probe
[
FileParsingTaskSignal
]
private
val
testRequests
=
TestSink
.
probe
[
FileParsingResultSignal
]
private
val
testRequests
=
TestSink
.
probe
[
FileParsingResultSignal
]
val
testGraph
=
RunnableGraph
.
fromGraph
(
private
val
testGraph
=
RunnableGraph
.
fromGraph
(
GraphDSL
.
create
(
testInput
,
testRequests
)((
_
,
_
))
{
implicit
builder
=>
(
source
,
sink
)
=>
GraphDSL
.
create
(
testInput
,
testRequests
)((
_
,
_
))
{
implicit
builder
=>
(
source
,
sink
)
=>
import
GraphDSL.Implicits._
import
GraphDSL.Implicits._
...
@@ -84,19 +75,19 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -84,19 +75,19 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
"a CalculationParsingFlow"
when
{
"a CalculationParsingFlow"
when
{
"managing a single worker"
should
{
"managing a single worker"
should
{
"expect a file tree started signal before any incoming parsing requests"
in
new
GraphWithDummyWorkers
(
1
)
{
"expect a file tree started signal before any incoming parsing requests"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
input1
)
source
.
sendNext
(
input1
)
sink
.
ensureSubscription
().
request
(
1
)
sink
.
ensureSubscription
().
request
(
1
)
sink
.
expectError
()
sink
.
expectError
()
}
}
"forward a file tree started signal after processing it"
in
new
GraphWithDummyWorkers
(
1
)
{
"forward a file tree started signal after processing it"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
sink
.
ensureSubscription
().
request
(
1
)
sink
.
ensureSubscription
().
request
(
1
)
sink
.
expectNext
(
FileParsingResultStartTree
(
sampleTreeScan
))
sink
.
expectNext
(
FileParsingResultStartTree
(
sampleTreeScan
))
}
}
"emit parsing results for every incoming parsing request in order of input"
in
new
GraphWithDummyWorkers
(
1
)
{
"emit parsing results for every incoming parsing request in order of input"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
)
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
)
sink
.
ensureSubscription
().
request
(
4
)
sink
.
ensureSubscription
().
request
(
4
)
...
@@ -112,7 +103,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -112,7 +103,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
}
}
}
}
"forward a file tree finished signal after processing it"
in
new
GraphWithDummyWorkers
(
1
)
{
"forward a file tree finished signal after processing it"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
))
sink
.
ensureSubscription
().
request
(
2
)
sink
.
ensureSubscription
().
request
(
2
)
...
@@ -120,7 +111,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -120,7 +111,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
sink
.
expectNext
(
FileParsingResultEndTree
(
sampleTreeScan
))
sink
.
expectNext
(
FileParsingResultEndTree
(
sampleTreeScan
))
}
}
"reject parsing requests after receiving the tree finished signal"
in
new
GraphWithDummyWorkers
(
1
)
{
"reject parsing requests after receiving the tree finished signal"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
)).
sendNext
(
input1
)
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
)).
sendNext
(
input1
)
sink
.
ensureSubscription
().
request
(
3
)
sink
.
ensureSubscription
().
request
(
3
)
...
@@ -128,7 +119,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -128,7 +119,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
sink
.
expectError
()
sink
.
expectError
()
}
}
"complete the stage if upstream signals completion before all elements are processed"
in
new
GraphWithDummyWorkers
(
1
)
{
"complete the stage if upstream signals completion before all elements are processed"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
).
sendComplete
()
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
).
sendComplete
()
sink
.
ensureSubscription
().
request
(
4
)
sink
.
ensureSubscription
().
request
(
4
)
...
@@ -136,7 +127,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -136,7 +127,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
sink
.
expectComplete
()
sink
.
expectComplete
()
}
}
"complete the stage if upstream signals completion after all elements are processed"
in
new
GraphWithDummyWorkers
(
1
)
{
"complete the stage if upstream signals completion after all elements are processed"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
)
source
.
sendNext
(
input1
).
sendNext
(
input2
).
sendNext
(
input3
)
sink
.
ensureSubscription
().
request
(
4
)
sink
.
ensureSubscription
().
request
(
4
)
...
@@ -145,14 +136,14 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -145,14 +136,14 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
sink
.
expectComplete
()
sink
.
expectComplete
()
}
}
"unpack a zip archive when the start tree signal arrives"
in
new
GraphWithDummyWorkers
(
1
)
{
"unpack a zip archive when the start tree signal arrives"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
sink
.
ensureSubscription
().
request
(
1
)
sink
.
ensureSubscription
().
request
(
1
)
sink
.
expectNext
()
sink
.
expectNext
()
verify
(
dummyArchiveHandler
).
extractZipArchive
(
sampleTreeScan
.
treeBasePath
)
verify
(
dummyArchiveHandler
).
extractZipArchive
(
sampleTreeScan
.
treeBasePath
)
}
}
"delete the extracted files from a zip archive when the end tree signal arrives"
in
new
GraphWithDummyWorkers
(
1
)
{
"delete the extracted files from a zip archive when the end tree signal arrives"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)))
sink
.
ensureSubscription
().
request
(
2
)
sink
.
ensureSubscription
().
request
(
2
)
...
@@ -160,7 +151,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -160,7 +151,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
verify
(
dummyArchiveHandler
).
cleanUpExtractedArchive
(
sampleTreeScan
.
treeBasePath
)
verify
(
dummyArchiveHandler
).
cleanUpExtractedArchive
(
sampleTreeScan
.
treeBasePath
)
}
}
"not interact with the ArchiveHandler when processing directory file trees"
in
new
GraphWithDummyWorkers
(
1
)
{
"not interact with the ArchiveHandler when processing directory file trees"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskEndTree
(
sampleTreeScan
))
sink
.
ensureSubscription
().
request
(
2
)
sink
.
ensureSubscription
().
request
(
2
)
...
@@ -168,7 +159,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -168,7 +159,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
verifyZeroInteractions
(
dummyArchiveHandler
)
verifyZeroInteractions
(
dummyArchiveHandler
)
}
}
"specify the location of the main file to the parsing engine when handling directories"
in
new
GraphWithDummyWorkers
(
1
)
{
"specify the location of the main file to the parsing engine when handling directories"
in
new
Fixture
(
1
)
{
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
FileParsingTaskStartTree
(
sampleTreeScan
))
source
.
sendNext
(
input1
)
source
.
sendNext
(
input1
)
sink
.
ensureSubscription
().
request
(
2
).
expectNextN
(
2
)
sink
.
ensureSubscription
().
request
(
2
).
expectNextN
(
2
)
...
@@ -176,10 +167,9 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -176,10 +167,9 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
verify
(
dummyWorkers
.
head
).
processRequest
(
any
(),
ArgumentMatchers
.
eq
(
expectedPath
))
verify
(
dummyWorkers
.
head
).
processRequest
(
any
(),
ArgumentMatchers
.
eq
(
expectedPath
))
}
}
"specify the temporarily extracted main file to the parsing engine when handling zip archives"
in
new
GraphWithDummyWorkers
(
1
)
{
"specify the temporarily extracted main file to the parsing engine when handling zip archives"
in
new
Fixture
(
1
)
{
val
zipTreeTask
=
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)
val
zipTreeTask
=
sampleTreeScan
.
copy
(
treeType
=
TreeType
.
Zip
)
val
extractedPath
=
Paths
.
get
(
"/tmp/extracted/magic"
)
val
extractedPath
=
Paths
.
get
(
"/tmp/extracted/magic"
)
// val task = aFileParsingTask().withExtractedPath(Some(extractedPath)).build()
when
(
dummyArchiveHandler
.
extractZipArchive
(
any
())).
thenReturn
(
extractedPath
)
when
(
dummyArchiveHandler
.
extractZipArchive
(
any
())).
thenReturn
(
extractedPath
)
source
.
sendNext
(
FileParsingTaskStartTree
(
zipTreeTask
))
source
.
sendNext
(
FileParsingTaskStartTree
(
zipTreeTask
))
source
.
sendNext
(
input1
.
copy
(
treeTask
=
zipTreeTask
))
source
.
sendNext
(
input1
.
copy
(
treeTask
=
zipTreeTask
))
...
@@ -188,7 +178,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -188,7 +178,7 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
verify
(
dummyWorkers
.
head
).
processRequest
(
any
(),
ArgumentMatchers
.
eq
(
expectedPath
))
verify
(
dummyWorkers
.
head
).
processRequest
(
any
(),
ArgumentMatchers
.
eq
(
expectedPath
))
}
}
"gracefully fail parsing requests with unknown or not supported file tree types"
in
new
GraphWithDummyWorkers
(
1
)
{
"gracefully fail parsing requests with unknown or not supported file tree types"
in
new
Fixture
(
1
)
{
sink
.
ensureSubscription
().
request
(
6
)
sink
.
ensureSubscription
().
request
(
6
)
Seq
(
TreeType
.
Unknown
,
TreeType
.
File
,
TreeType
.
Tar
).
foreach
{
treeType
=>
Seq
(
TreeType
.
Unknown
,
TreeType
.
File
,
TreeType
.
Tar
).
foreach
{
treeType
=>
val
anotherTreeScan
=
sampleTreeScan
.
copy
(
treeType
=
treeType
)
val
anotherTreeScan
=
sampleTreeScan
.
copy
(
treeType
=
treeType
)
...
@@ -203,8 +193,8 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
...
@@ -203,8 +193,8 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
}
}
}
}
"handle parsing requests with supported file tree types"
in
new
GraphWithDummyWorkers
(
1
)
{
"handle parsing requests with supported file tree types"
in
new
Fixture
(
1
)
{
when
(
dummyArchiveHandler
.
extractZipArchive
(
any
())).
thenReturn
(
unusedPath
)
when
(
dummyArchiveHandler
.
extractZipArchive
(
any
())).
thenReturn
(
Paths
.
get
(
"/tmp/extracted/magic"
)
)
sink
.
ensureSubscription
().
request
(
4
)
sink
.
ensureSubscription
().
request
(
4
)
Seq
(
TreeType
.
Directory
,
TreeType
.
Zip
).
foreach
{
treeType
=>
Seq
(
TreeType
.
Directory
,
TreeType
.
Zip
).
foreach
{
treeType
=>
val
anotherTreeScan
=
sampleTreeScan
.
copy
(
treeType
=
treeType
)
val
anotherTreeScan
=
sampleTreeScan
.
copy
(
treeType
=
treeType
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment