Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
esde
machine-learning
AMBS
Commits
a6121ba2
Commit
a6121ba2
authored
Feb 03, 2021
by
Michael Langguth
Browse files
Add save_gpus_info to set-up mof model.
parent
aa6d9dd6
Pipeline
#58798
passed with stages
in 2 minutes and 6 seconds
Changes
1
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
video_prediction_tools/main_scripts/main_train_models.py
View file @
a6121ba2
...
...
@@ -73,6 +73,7 @@ class TrainModel(object):
self
.
create_saver_and_writer
()
self
.
setup_gpu_config
()
self
.
calculate_samples_and_epochs
()
self
.
save_gpus_info
()
print
(
"setup done"
)
def
set_seed
(
self
):
...
...
@@ -285,9 +286,9 @@ class TrainModel(object):
#print("hvd_size:",hvd.size())
#print("hvd_local_rank:",hvd.local_rank())
# also track computing node
cnode_file
=
os
.
path
.
join
(
self
.
output_dir
,
"GPU_worker{0}.json"
.
format
(
str
(
hvd
.
local_
rank
())))
cnode_file
=
os
.
path
.
join
(
self
.
output_dir
,
"GPU_worker{0}.json"
.
format
(
str
(
hvd
.
rank
())))
with
open
(
cnode_file
,
"w"
)
as
fjs
:
json
.
dump
({
"worker{0}"
.
format
(
str
(
hvd
.
local_
rank
())):
host
},
fjs
)
json
.
dump
({
"worker{0}"
.
format
(
str
(
hvd
.
rank
())):
host
},
fjs
)
def
save_timing_to_pkl
(
self
,
training_time
,
time_per_iteration
):
"""
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment