-
Notifications
You must be signed in to change notification settings - Fork 2
/
benchmark_config.yml.example
130 lines (126 loc) · 4.14 KB
/
benchmark_config.yml.example
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# You need have access to a GalaxyInstance with admin-rights
galaxy:
url: "http://galaxy.example.com"
user_key: "blablabla"
# Check, if tools need to be installed when running GalaxyWorkflow (can be turned off after first benchmark-run)
shed_install: true
# Should Galaxy be configured to use the given Destinations or is everything already set?
configure_job_destinations: true
# Used to deploy DynamicDestinations via Ansible
ssh_user: ubuntu
ssh_key: /local/path/to/ssh/key.cert
galaxy_root_path: /srv/galaxy
galaxy_config_dir: /srv/galaxy/server/config
galaxy_user: galaxy
# Used for analyzing results
influxdb:
host: influxdb.example.com
port: 8086
username: glx_benchmarker_user
password: supersecret
db_name: glx_benchmarker
# Configure all the Destinations that should be benchmarked
destinations:
- name: LocalPulsar
type: PulsarMQ
amqp_url: "pyamqp://username:[email protected]:5672//"
# If used for ColdWarmBenchmark, we need to have ssh-access to the Pulsar-Server
host: pulsar.example.com
host_user: centos
ssh_key: /local/path/to/ssh/key.cert
tool_dependency_dir: /data/share/tools
jobs_directory_dir: /data/share/staging
persistence_dir: /data/share/persisted_data
# To configure additional params in job_conf.xml
job_plugin_params:
manager: __default__
job_destination_params:
dependency_resolution: remote
default_file_action: remote_transfer
remote_metadata: false
rewrite_parameters: true
amqp_acknowledge: true
amqp_ack_republish_time: 10
- name: RemotePulsar1
type: PulsarMQ
# If credentials are set, Destination won't create a user on Galaxy and will just use these for submitting jobs
galaxy_user_name: user2
galaxy_user_key: blablablakey
- name: RemotePulsar1
type: PulsarMQ
amqp_url: "pyamqp://username:[email protected]:5672//"
job_plugin_params:
manager: __default__
job_destination_params:
dependency_resolution: remote
default_file_action: remote_transfer
remote_metadata: false
rewrite_parameters: true
amqp_acknowledge: true
amqp_ack_republish_time: 10
- name: CondorManager
type: Condor
host: condor-manager.uni.andreas-sk.de
host_user: centos
ssh_key: /local/path/to/ssh/key.cert
jobs_directory_dir: /data/share/condor
workflows:
- name: GalaxyWorkflow1
type: Galaxy
path: path/to/galaxy/workflow/file.ga
timeout: 100 # Optional. Workflow will be canceled after timeout.
- name: CondorWorkflow1
type: Condor
path: path/to/condor/workflow/folder
job_file: job.job # needs to be in directory at "path"
benchmarks:
- name: ColdvsWarm
type: ColdvsWarm
destinations:
- LocalPulsar
runs_per_workflow: 5
workflows:
- GalaxyWorkflow1
# Pre task to clean up Pulsar, so the workflow-run is actually cold (i.e. run for the "first time")
cold_pre_task:
type: "ansible-playbook"
playbook: "coldwarm_pretask.yml"
warm_pre_task:
type: "ansible-playbook"
playbook: "cleanup-pulsar.yml"
# If you want to clean up Pulsar after Benchmark ran
post_task:
type: "ansible-playbook"
playbook: "cleanup_pulsar.yml"
- name: DestinationComparison
type: DestinationComparison
pre_task:
type: "ansible-playbook"
playbook: "cleanup-pulsar.yml"
post_task:
type: "ansible-playbook"
playbook: "cleanup-pulsar.yml" # Call some script after ending benchmark (like for cleaning up)
destinations:
- LocalPulsar
- RemotePulsar1
- RemotePulsar2
runs_per_workflow: 5
warmup: true # Should a warmup-run happen before the actual benchmarking?
workflows:
- GalaxyWorkflow1
- name: PulsarBurstBenchmark
type: Burst
runs_per_workflow: 100
burst_rate: 1 # How many workflows should be submitted per second
destinations:
- RemotePulsar1 # Only one destination allowed!
workflows:
- GalaxyWorkflow1
- name: CondorBurstBenchmark
type: Burst
runs_per_workflow: 1000
burst_rate: 20
destinations:
- CondorManager
workflows:
- CondorWorkflow1