-
Notifications
You must be signed in to change notification settings - Fork 6
/
Copy pathrun_all.py
86 lines (72 loc) · 3.23 KB
/
run_all.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import os, subprocess, time, sys, json
lair_dir = "/home/lair"
bears_dir = "/home/bears_analyses"
max_processes = 4
try:
git_pull = subprocess.check_output(["git", "pull"], cwd=bears_dir)
except subprocess.CalledProcessError as git_pull_err:
sys.exit("Return code for '" + str(" ".join(git_pull_err.cmd)) + "' was " + str(git_pull_err.returncode) + "\n")
old_pids = {}
if os.path.isfile('PIDS.json'):
with open('PIDS.json', 'r') as jf:
old_pids = json.load(jf)
directories = next(os.walk('.'))[1]
a_dirs = []
for directory in directories:
if ('config.json' in os.listdir(directory)):
if (directory not in old_pids):
a_dirs.append(directory)
process_ids = {}
ret_codes = {}
for directory in a_dirs:
if ('so.rds' not in os.listdir(directory)):
num_open = 0
for k, v in process_ids.iteritems():
if v.poll is None:
num_open += 1
if num_open > 3:
ret_codes[k] = v.wait()
num_open -= 1
process_ids[directory] = subprocess.Popen(["snakemake", "-p", "--configfile", directory + "/config.json"])
process_file = open('PIDS.json', 'w')
process_file.write('{\n')
time_opened = time.strftime("%c")
for k, v in process_ids.iteritems():
process_file.write('\t"' + k + '": "' + str(v.pid) + '",\n')
process_file.write('\t"time-opened": "' + time_opened + '"\n')
process_file.write('}')
process_file.close()
for k, v in process_ids.iteritems():
if k not in ret_codes:
ret_codes[k] = v.wait()
with open(k + '/config.json') as configf:
config = json.load(configf)
if 'analysis' in config:
srv_dir = config['analysis']
base = "http://lair.berkeley.edu/"
base_index = srv_dir.find(base) + len(base)
srv_dir = srv_dir[base_index:]
srv_dir = "/srv/shiny_server/" + srv_dir
try: #requires user input for sudo access
sudo_deploy = subprocess.check_output(["sudo", "mkdir", "-p", srv_dir])
sudo_mv = subprocess.check_output(["sudo", "cp", k + "/so.rds", k + "/app.R", srv_dir])
sudo_sed = subprocess.check_output(["sudo", "sed", "-i", """'1i .libPaths(c(.libPaths(), "/home/psturm/R/x86_64-pc-linux-gnu-library/3.3"))'""", srv_dir + "app.R"])
except subprocess.CalledProcessError as git_pull_err:
sys.exit("Return code for '" + str(" ".join(git_pull_err.cmd)) + "' was " + str(git_pull_err.returncode) + "\n")
time_closed = time.strftime("%c")
process_file = open('PIDS.json', 'w')
process_file.write('{\n')
process_file.write('\t"time-opened": "' + time_opened + '",\n')
for k, v in ret_codes.iteritems():
process_file.write('\t"' + k + '": ' + str(v) + ',\n')
process_file.write('\t"time-closed": "' + time_closed + '"\n')
process_file.write('}')
process_file.close()
try:
git_pull = subprocess.check_output(["git", "pull"], cwd=lair_dir)
update_dois = subprocess.check_output(["python", "grab_doi_metadata.py"], cwd=lair_dir+"/_data")
git_add = subprocess.check_output(["git", "add", "--all"], cwd=lair_dir)
git_commit = subprocess.checkoutput(['git', 'commit', '-m', '"auto-commit; close time ' + time_closed + '"'], cwd=lair_dir)
git_push = subprocess.checkoutput(['git', 'push', 'origin', 'gh-pages'], cwd=lair_dir) #requires user input
except subprocess.CalledProcessError as git_pull_err:
sys.exit("Return code for '" + str(" ".join(git_pull_err.cmd)) + "' was " + str(git_pull_err.returncode) + "\n")