File scripts/builder.php changed (mode: 100644) (index 6250577..f4d42a6) |
... |
... |
function xnew($key, $arg) |
77 |
77 |
/* |
/* |
78 |
78 |
* Returns the number of active jobs for a worker |
* Returns the number of active jobs for a worker |
79 |
79 |
*/ |
*/ |
80 |
|
function worker_active_jobs($wid) |
|
|
80 |
|
function worker_active_jobs($worker_id) |
81 |
81 |
{ |
{ |
82 |
82 |
global $jobs; |
global $jobs; |
83 |
83 |
|
|
84 |
84 |
$ret = 0; |
$ret = 0; |
85 |
|
foreach ($jobs as $jid => $info) |
|
86 |
|
if ($info['worker_id'] == $wid) |
|
|
85 |
|
foreach ($jobs as $jid => $job) { |
|
86 |
|
if (!isset($job['worker_id'])) { |
|
87 |
|
rg_log($jid . ': DEBUG: worker_id is not defined!'); |
|
88 |
|
continue; |
|
89 |
|
} |
|
90 |
|
|
|
91 |
|
if ($job['worker_id'] == $worker_id) |
87 |
92 |
$ret++; |
$ret++; |
|
93 |
|
} |
88 |
94 |
|
|
89 |
95 |
return $ret; |
return $ret; |
90 |
96 |
} |
} |
|
... |
... |
function xdispatch_one($key, $data) |
164 |
170 |
|
|
165 |
171 |
$now = time(); |
$now = time(); |
166 |
172 |
$s = &$rg_conns[$key]; |
$s = &$rg_conns[$key]; |
167 |
|
//rg_log('DEBUG: data=' . $data); |
|
|
173 |
|
//rg_log_debug('data=' . $data); |
168 |
174 |
//rg_log_ml($key . ': DEBUG: xdispatch_one s: ' . rg_array2string($s)); |
//rg_log_ml($key . ': DEBUG: xdispatch_one s: ' . rg_array2string($s)); |
169 |
175 |
while (1) { |
while (1) { |
170 |
176 |
$u = @json_decode($data, TRUE); |
$u = @json_decode($data, TRUE); |
|
... |
... |
function xdispatch_one($key, $data) |
300 |
306 |
|
|
301 |
307 |
// The rest of the commands need a job id |
// The rest of the commands need a job id |
302 |
308 |
if (!isset($u['id'])) { |
if (!isset($u['id'])) { |
303 |
|
$err = array('op' => $op, 'errstr' => 'protocol error: job id not found'); |
|
|
309 |
|
$m = 'protocol error: field \'id\' not found'; |
|
310 |
|
rg_log($key . ': ' . $m); |
|
311 |
|
$err = array('op' => $op, 'errstr' => $m); |
304 |
312 |
rg_conn_enq($key, @json_encode($err) . "\n"); |
rg_conn_enq($key, @json_encode($err) . "\n"); |
305 |
313 |
rg_conn_shutdown($key, 2); |
rg_conn_shutdown($key, 2); |
306 |
314 |
break; |
break; |
307 |
315 |
} |
} |
308 |
|
|
|
309 |
316 |
$jid = $u['id']; |
$jid = $u['id']; |
|
317 |
|
|
|
318 |
|
if (!isset($jobs[$jid])) { |
|
319 |
|
rg_log($jid . ': job not found. Probably an old one. Sending DRE.'); |
|
320 |
|
$a = array('op' => 'DRE', 'id' => $jid); |
|
321 |
|
rg_conn_enq($key, @json_encode($a) . "\n"); |
|
322 |
|
break; |
|
323 |
|
} |
310 |
324 |
$job = &$jobs[$jid]; |
$job = &$jobs[$jid]; |
311 |
325 |
|
|
312 |
326 |
// security check |
// security check |
313 |
|
if ($job['worker_id'] != $s['worker_id']) { |
|
|
327 |
|
if (!isset($job['worker_id'])) |
|
328 |
|
rg_log_ml($jid . ': DEBUG: NO WORKER_ID: job: ' . rg_array2string($job)); |
|
329 |
|
if (!isset($job['worker_id']) |
|
330 |
|
|| ($job['worker_id'] != $s['worker_id'])) { |
314 |
331 |
$m = 'job not associated with worker'; |
$m = 'job not associated with worker'; |
315 |
332 |
rg_log($key . ': ' . $jid . ': error: ' . $m); |
rg_log($key . ': ' . $jid . ': error: ' . $m); |
316 |
|
rg_log($key . ': ' . $jid . ': job[worker_id]=' . $job['worker_id']); |
|
|
333 |
|
rg_log($key . ': ' . $jid . ': job[worker_id]=' . isset($job['worker_id']) ? $job['worker_id'] : '-'); |
317 |
334 |
rg_log($key . ': ' . $jid . ': s[worker_id]=' . $s['worker_id']); |
rg_log($key . ': ' . $jid . ': s[worker_id]=' . $s['worker_id']); |
318 |
335 |
$err = array('op' => 'abort_job', 'jid' => $jid, 'errstr' => $m); |
$err = array('op' => 'abort_job', 'jid' => $jid, 'errstr' => $m); |
319 |
336 |
rg_conn_enq($key, json_encode($err) . "\n"); |
rg_conn_enq($key, json_encode($err) . "\n"); |
320 |
337 |
break; |
break; |
321 |
338 |
} |
} |
322 |
339 |
|
|
|
340 |
|
if (strcmp($op, 'rg_notify') == 0) { |
|
341 |
|
$m = isset($u['m']) ? $u['m'] : 'MISSING'; |
|
342 |
|
rg_log($key . ': ' . $jid . ': rg_notify: ' . $m); |
|
343 |
|
break; |
|
344 |
|
} |
|
345 |
|
|
323 |
346 |
if (strcmp($op, 'STA') == 0) { |
if (strcmp($op, 'STA') == 0) { |
324 |
347 |
$job['worker_started'] = $now; |
$job['worker_started'] = $now; |
325 |
348 |
job_save($job); |
job_save($job); |
|
... |
... |
function xdispatch_one($key, $data) |
333 |
356 |
$send_confirmation = TRUE; |
$send_confirmation = TRUE; |
334 |
357 |
if (!empty($u['error'])) { |
if (!empty($u['error'])) { |
335 |
358 |
// TODO: we need to distinguish between fatal errors and transient errors |
// TODO: we need to distinguish between fatal errors and transient errors |
|
359 |
|
if (!isset($job['tries'])) |
|
360 |
|
$job['tries'] = 0; |
|
361 |
|
$job['tries']++; |
|
362 |
|
rg_log($key . ': ' . $jid . ': job tries: ' . $job['tries']); |
|
363 |
|
|
336 |
364 |
rg_log($key . ': ' . $jid . ': job failed with error: ' . $u['error']); |
rg_log($key . ': ' . $jid . ': job failed with error: ' . $u['error']); |
337 |
365 |
rg_log($key . ': ' . $jid . ': job failed with error2: ' . $u['error2']); |
rg_log($key . ': ' . $jid . ': job failed with error2: ' . $u['error2']); |
338 |
366 |
// Delay job and retry (on another worker) |
// Delay job and retry (on another worker) |
339 |
|
$job['next_try'] = $now + 60; |
|
|
367 |
|
$job['next_try'] = $now + $job['tries'] * 600; |
340 |
368 |
$k = $s['worker_id']; |
$k = $s['worker_id']; |
341 |
|
$job['avoid'][$k] = 1; |
|
|
369 |
|
$job['avoid'][$k] = time() + 12 * 3600; |
342 |
370 |
$job['worker_id'] = 0; |
$job['worker_id'] = 0; |
343 |
371 |
job_save($job); |
job_save($job); |
344 |
372 |
} else { |
} else { |
|
... |
... |
function xdispatch_one($key, $data) |
539 |
567 |
$meta['map_into_source'][] = $path; |
$meta['map_into_source'][] = $path; |
540 |
568 |
} |
} |
541 |
569 |
} else { |
} else { |
542 |
|
rg_log_ml('DEBUG: para does not have map_into_source: ' . rg_array2string($para)); |
|
|
570 |
|
rg_log_debug('para does not have map_into_source: ' . rg_array2string($para)); |
543 |
571 |
} |
} |
544 |
572 |
|
|
545 |
573 |
foreach ($para['map'] as $path) { |
foreach ($para['map'] as $path) { |
|
... |
... |
function xdispatch_one($key, $data) |
632 |
660 |
} |
} |
633 |
661 |
unset($job); |
unset($job); |
634 |
662 |
|
|
635 |
|
//rg_log_ml('DEBUG: s: ' . rg_array2string($s)); |
|
|
663 |
|
//rg_log_debug('s: ' . rg_array2string($s)); |
636 |
664 |
unset($s); |
unset($s); |
637 |
665 |
} |
} |
638 |
666 |
|
|
|
... |
... |
function xdispatch($key, $data) |
652 |
680 |
continue; |
continue; |
653 |
681 |
} |
} |
654 |
682 |
|
|
655 |
|
//rg_log('DEBUG: searching for \n in ' . substr($data, 0, 100) . '...'); |
|
|
683 |
|
//rg_log_debug('searching for \n in ' . substr($data, 0, 100) . '...'); |
656 |
684 |
$pos = strpos($data, "\n"); |
$pos = strpos($data, "\n"); |
657 |
685 |
if ($pos === FALSE) |
if ($pos === FALSE) |
658 |
686 |
return $ret; |
return $ret; |
|
... |
... |
function rg_process_job($db, &$job) |
681 |
709 |
} |
} |
682 |
710 |
|
|
683 |
711 |
// Should we delay because of a previous fail? |
// Should we delay because of a previous fail? |
684 |
|
if (isset($job['next_try']) && ($job['next_try'] < time())) { |
|
685 |
|
rg_log($jid . ': job is suspended till ' . date('Ym-m-d H:i:s', $job['next_try'])); |
|
|
712 |
|
if (isset($job['next_try']) && ($job['next_try'] > time())) { |
|
713 |
|
rg_log($jid . ': job is suspended till ' |
|
714 |
|
. date('Y-m-d H:i:s', $job['next_try'])); |
686 |
715 |
return; |
return; |
687 |
716 |
} |
} |
688 |
717 |
|
|
|
... |
... |
function rg_process_job($db, &$job) |
701 |
730 |
$job['next_try'] = time() + 60; |
$job['next_try'] = time() + 60; |
702 |
731 |
return; |
return; |
703 |
732 |
} |
} |
704 |
|
//rg_log_ml('DEBUG: workers list: ' . rg_array2string($workers_list)); |
|
|
733 |
|
//rg_log_debug('workers list: ' . rg_array2string($workers_list)); |
705 |
734 |
|
|
706 |
735 |
// Trying to find a worker in the list of connections |
// Trying to find a worker in the list of connections |
707 |
736 |
foreach ($rg_conns as $key => &$i) { |
foreach ($rg_conns as $key => &$i) { |
|
... |
... |
function rg_process_job($db, &$job) |
733 |
762 |
} |
} |
734 |
763 |
$wi = $workers_list[$k]; |
$wi = $workers_list[$k]; |
735 |
764 |
|
|
736 |
|
if (isset($job['avoid'][$k])) { |
|
|
765 |
|
if (isset($job['avoid'][$k]) && ($job['avoid'][$k] > time())) { |
737 |
766 |
rg_log($key . ': ' . $jid . ': we must avoid worker ' . $name); |
rg_log($key . ': ' . $jid . ': we must avoid worker ' . $name); |
738 |
767 |
continue; |
continue; |
739 |
768 |
} |
} |
|
... |
... |
function rg_process_job($db, &$job) |
749 |
778 |
} |
} |
750 |
779 |
} |
} |
751 |
780 |
|
|
|
781 |
|
$env_found = FALSE; |
752 |
782 |
foreach ($i['ann']['env'] as $env => $junk) { |
foreach ($i['ann']['env'] as $env => $junk) { |
753 |
783 |
if (strcasecmp($req['env'], $env) != 0) { |
if (strcasecmp($req['env'], $env) != 0) { |
754 |
|
rg_log($key . ': ' . $jid . ': DEBUG: worker ' . $name |
|
755 |
|
. ': job env [' . $req['env'] . ']' |
|
756 |
|
. ' != worker [' . $env . ']'); |
|
|
784 |
|
//rg_log($key . ': ' . $jid . ': DEBUG: worker ' . $name |
|
785 |
|
// . ': job env [' . $req['env'] . ']' |
|
786 |
|
// . ' != worker [' . $env . ']'); |
757 |
787 |
continue; |
continue; |
758 |
788 |
} |
} |
759 |
789 |
|
|
760 |
|
// Send only what is really needed |
|
761 |
|
$job2 = array(); |
|
762 |
|
$job2['op'] = 'BLD'; |
|
763 |
|
$job2['id'] = $jid; |
|
764 |
|
$job2['cmds'] = $req['cmds']; |
|
765 |
|
if (isset($req['secrets'])) |
|
766 |
|
$job2['secrets'] = $req['secrets']; |
|
767 |
|
$job2['packages'] = $req['packages']; |
|
768 |
|
$job2['hook_id'] = $req['hook_id']; |
|
769 |
|
$job2['url'] = $req['url']; |
|
770 |
|
$job2['head'] = $req['head']; |
|
771 |
|
$job2['env'] = $req['env']; |
|
772 |
|
rg_conn_enq($key, json_encode($job2) . "\n"); |
|
773 |
|
|
|
774 |
|
$job['worker_id'] = $i['worker_id']; |
|
775 |
|
$job['worker_name'] = $name; |
|
776 |
|
$job['worker_started'] = 0; |
|
777 |
|
$job['worker_sent'] = time(); |
|
778 |
|
|
|
779 |
|
//rg_log_ml('DEBUG: After sending BLD: job: ' . print_r($job, TRUE)); |
|
780 |
|
// TODO: after some time, if worker_started is still 0, |
|
781 |
|
// set the 'worker_id' to 0 to be able to go in other place |
|
782 |
|
// TODO: maybe the client must resync with server to |
|
783 |
|
// abort jobs already done on another host, to not |
|
784 |
|
// duplicate work |
|
785 |
|
rg_log($key . ': ' . $jid . ': job sent to worker [' . $name . ']'); |
|
786 |
|
return; |
|
|
790 |
|
$env_found = TRUE; |
|
791 |
|
break; |
787 |
792 |
} |
} |
|
793 |
|
if (!$env_found) |
|
794 |
|
continue; |
|
795 |
|
|
|
796 |
|
// Send only what is really needed |
|
797 |
|
$job2 = $req; |
|
798 |
|
|
|
799 |
|
// TODO: this should be configurable! |
|
800 |
|
$job2['rgfs_server'] = 'rg.embedromix.ro'; |
|
801 |
|
$job2['rgfs_port'] = 9003; |
|
802 |
|
$job2['rgfs_url'] = '/rgfs'; |
|
803 |
|
|
|
804 |
|
$job2['worker_id'] = $i['worker_id']; |
|
805 |
|
$job2['worker_name'] = $name; |
|
806 |
|
|
|
807 |
|
$job2['op'] = 'BLD'; |
|
808 |
|
$job2['id'] = $jid; |
|
809 |
|
rg_conn_enq($key, json_encode($job2) . "\n"); |
|
810 |
|
|
|
811 |
|
$job['worker_id'] = $i['worker_id']; |
|
812 |
|
$job['worker_name'] = $name; |
|
813 |
|
$job['worker_started'] = 0; |
|
814 |
|
$job['worker_sent'] = time(); |
|
815 |
|
|
|
816 |
|
//rg_log_debug('After sending BLD: job: ' . rg_array2string($job)); |
|
817 |
|
// TODO: after some time, if worker_started is still 0, |
|
818 |
|
// set the 'worker_id' to 0 to be able to go in other place |
|
819 |
|
// TODO: maybe the client must resync with server to |
|
820 |
|
// abort jobs already done on another host, to not |
|
821 |
|
// duplicate work |
|
822 |
|
rg_log($key . ': ' . $jid . ': job sent to worker [' . $name . ']'); |
|
823 |
|
rg_log_debug('sent job: ' . rg_array2string($job2)); |
|
824 |
|
return; |
788 |
825 |
} |
} |
789 |
826 |
unset($i); |
unset($i); |
790 |
827 |
|
|
|
... |
... |
rg_prof_start('MAIN'); |
797 |
834 |
rg_log_set_file($rg_log_dir . '/builder.log'); |
rg_log_set_file($rg_log_dir . '/builder.log'); |
798 |
835 |
rg_log_set_sid('000000'); // to spread the logs |
rg_log_set_sid('000000'); // to spread the logs |
799 |
836 |
|
|
800 |
|
rg_lock_or_exit('builder.lock'); |
|
|
837 |
|
$rg_cache_core_enable = FALSE; |
|
838 |
|
|
|
839 |
|
rg_lock_or_exit('builder.lock', 0); |
801 |
840 |
|
|
802 |
841 |
rg_log('Start (ver=' . $rocketgit_version . ')...'); |
rg_log('Start (ver=' . $rocketgit_version . ')...'); |
803 |
842 |
|
|
|
... |
... |
if ($db === FALSE) { |
825 |
864 |
if (rg_struct_ok($db) === FALSE) |
if (rg_struct_ok($db) === FALSE) |
826 |
865 |
exit(0); |
exit(0); |
827 |
866 |
|
|
|
867 |
|
rg_debug_set(rg_state_get($db, 'debug')); |
|
868 |
|
|
828 |
869 |
// Prepare socket |
// Prepare socket |
829 |
870 |
$socket = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP); |
$socket = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP); |
830 |
871 |
if ($socket === FALSE) { |
if ($socket === FALSE) { |
|
... |
... |
$rg_conns['master']['func_new'] = 'xnew'; |
852 |
893 |
$rg_conns['master']['func_new_arg'] = $db; |
$rg_conns['master']['func_new_arg'] = $db; |
853 |
894 |
|
|
854 |
895 |
// What features the builder supports |
// What features the builder supports |
855 |
|
$features = array('allow_stats' => 1); |
|
|
896 |
|
$features = array( |
|
897 |
|
'allow_stats' => 1, |
|
898 |
|
'notify' => 1); |
856 |
899 |
|
|
857 |
900 |
$workers = 0; |
$workers = 0; |
858 |
901 |
$original_mtime = @filemtime(__FILE__); |
$original_mtime = @filemtime(__FILE__); |
859 |
902 |
do { |
do { |
860 |
|
// We do not want stale entries! |
|
861 |
|
rg_cache_core_destroy(); |
|
862 |
|
|
|
863 |
903 |
// Check our mtime so we can upgrade the software and this script |
// Check our mtime so we can upgrade the software and this script |
864 |
904 |
// will restart. |
// will restart. |
865 |
905 |
clearstatcache(); |
clearstatcache(); |
File scripts/worker.php changed (mode: 100644) (index e23d3ae..c6336cf) |
... |
... |
require_once(__DIR__ . '/../inc/builder.inc.php'); |
23 |
23 |
require_once(__DIR__ . '/../inc/conn.inc.php'); |
require_once(__DIR__ . '/../inc/conn.inc.php'); |
24 |
24 |
require_once(__DIR__ . '/../inc/user/packages_rpm.inc.php'); |
require_once(__DIR__ . '/../inc/user/packages_rpm.inc.php'); |
25 |
25 |
|
|
26 |
|
rg_prof_start('MAIN'); |
|
27 |
|
|
|
28 |
|
if (!isset($_SERVER['argv'][1])) { |
|
29 |
|
$id = 'main'; |
|
30 |
|
} else { |
|
31 |
|
$id = $_SERVER['argv'][1]; |
|
32 |
|
} |
|
33 |
|
|
|
34 |
|
if (!isset($_SERVER['argv'][2])) { |
|
35 |
|
$conf_file = '/etc/rocketgit/worker.conf'; |
|
36 |
|
} else { |
|
37 |
|
$conf_file = $_SERVER['argv'][2]; |
|
38 |
|
} |
|
39 |
|
|
|
40 |
|
rg_log_set_file($rg_log_dir . '/worker-' . $id . '.log'); |
|
41 |
|
rg_log_set_sid("000000"); // to spread the logs |
|
42 |
|
|
|
43 |
26 |
|
|
44 |
27 |
function job_state($state) |
function job_state($state) |
45 |
28 |
{ |
{ |
|
... |
... |
function load_config($file) |
258 |
241 |
. ' -o Dpkg::Options::=--force-confnew install'; |
. ' -o Dpkg::Options::=--force-confnew install'; |
259 |
242 |
} else if (strcasecmp($x, 'fedora') == 0) { |
} else if (strcasecmp($x, 'fedora') == 0) { |
260 |
243 |
$i['pkg_cmd'] = 'dnf -y install'; |
$i['pkg_cmd'] = 'dnf -y install'; |
|
244 |
|
} else if (strcasecmp($x, 'rocky') == 0) { |
|
245 |
|
$i['pkg_cmd'] = 'dnf -y install'; |
261 |
246 |
} else if (strcasecmp($x, 'centos') == 0) { |
} else if (strcasecmp($x, 'centos') == 0) { |
262 |
247 |
if ($y > 7) |
if ($y > 7) |
263 |
248 |
$i['pkg_cmd'] = 'dnf -y install'; |
$i['pkg_cmd'] = 'dnf -y install'; |
264 |
249 |
else |
else |
265 |
250 |
$i['pkg_cmd'] = 'yum -y install'; |
$i['pkg_cmd'] = 'yum -y install'; |
|
251 |
|
} else if (strcasecmp($x, 'opensuse') == 0) { |
|
252 |
|
$i['pkg_cmd'] = 'zypper --non-interactive install'; |
|
253 |
|
} else if (strcasecmp($x, 'archlinux') == 0) { |
|
254 |
|
$i['pkg_cmd'] = 'pacman --sync --refresh --noconfirm --needed'; |
266 |
255 |
} else { |
} else { |
267 |
|
rg_log('Error! I do not know how to install packages on ' . $name); |
|
|
256 |
|
rg_log('Error! I do not know how to install packages on ' . $x |
|
257 |
|
. ' (name ' . $name . '). Ignore it.'); |
268 |
258 |
unset($conf['env'][$name]); |
unset($conf['env'][$name]); |
269 |
259 |
continue; |
continue; |
270 |
260 |
} |
} |
|
... |
... |
function rg_worker_libvirt_rights($f, $mode, $user, $group, &$reason, &$reason2) |
365 |
355 |
return $ret; |
return $ret; |
366 |
356 |
} |
} |
367 |
357 |
|
|
|
358 |
|
/* |
|
359 |
|
* Building common tools script |
|
360 |
|
* TODO: we should store E_CMD/E_OUTPUT somewhere and propagate it to builder. |
|
361 |
|
*/ |
|
362 |
|
function start_worker_build_tools($job, &$reason, &$reason2) |
|
363 |
|
{ |
|
364 |
|
$s = '' |
|
365 |
|
. '#!/bin/bash' . "\n" |
|
366 |
|
. "\n" |
|
367 |
|
. 'set -o pipefail' . "\n" |
|
368 |
|
. "\n" |
|
369 |
|
. '# Export global variables' . "\n" |
|
370 |
|
. 'export job_id=' . escapeshellarg($job['id']) . "\n" |
|
371 |
|
. 'export repo_username=' . escapeshellarg(isset($job['repo_username']) ? $job['repo_username'] : '') . "\n" |
|
372 |
|
. 'export job_uid=' . (isset($job['uid']) ? $job['uid'] : '') . "\n" |
|
373 |
|
. 'export env=' . escapeshellarg($job['env']) . "\n" |
|
374 |
|
. 'export env_distro=' . escapeshellarg($job['env_distro']) . "\n" |
|
375 |
|
. 'export env_major=' . escapeshellarg($job['env_major']) . "\n" |
|
376 |
|
. 'export env_arch=' . escapeshellarg($job['env_arch']) . "\n"; |
|
377 |
|
|
|
378 |
|
if (isset($job['rgfs_server'])) |
|
379 |
|
$s .= '' |
|
380 |
|
. 'export RGFS_SERVER=' . escapeshellarg($job['rgfs_server']) . "\n" |
|
381 |
|
. 'export RGFS_PORT=' . escapeshellarg($job['rgfs_port']) . "\n" |
|
382 |
|
. 'export RGFS_URL=' . escapeshellarg($job['rgfs_url']) . "\n" |
|
383 |
|
. 'export RGFS_DEBUG=100' . "\n" |
|
384 |
|
. 'export RGFS_name=' . 'wj' . $job['id'] . "\n" |
|
385 |
|
. 'export RGFS_head=' . escapeshellarg($job['head']) . "\n" |
|
386 |
|
. 'export RGFS_repo_id=' . escapeshellarg($job['repo_id']) . "\n" |
|
387 |
|
. 'export RGFS_uid=' . escapeshellarg($job['uid']) . "\n" |
|
388 |
|
. 'export RGFS_job_id=' . escapeshellarg($job['id']) . "\n" |
|
389 |
|
. 'export RGFS_env=' . escapeshellarg($job['env']) . "\n" |
|
390 |
|
. 'export RGFS_worker_id=' . escapeshellarg($job['worker_id']) . "\n" |
|
391 |
|
. 'export RGFS_worker_name=' . escapeshellarg($job['worker_name']) . "\n" |
|
392 |
|
. 'export RGFS_log_sid=' . $job['log_sid'] . "\n"; |
|
393 |
|
|
|
394 |
|
$s .= '' |
|
395 |
|
. "\n" |
|
396 |
|
. 'declare -gA pri2uid=()' . "\n"; |
|
397 |
|
|
|
398 |
|
$list = ''; $list_add = ''; |
|
399 |
|
if (isset($job['pkg_repos'])) { |
|
400 |
|
foreach ($job['pkg_repos'] as $pkg_repo_id => $info) { |
|
401 |
|
$list .= $list_add . $pkg_repo_id; $list_add = ' '; |
|
402 |
|
$s .= 'pri2uid[' . $pkg_repo_id . ']=' . $info['uid'] . "\n"; |
|
403 |
|
} |
|
404 |
|
} |
|
405 |
|
$s .= 'export pkg_repos_list="' . $list . '"' . "\n"; |
|
406 |
|
|
|
407 |
|
$s .= '' |
|
408 |
|
. '# Export ends' . "\n" |
|
409 |
|
. "\n" |
|
410 |
|
. 'E=0; E_CMD=""' . "\n" |
|
411 |
|
. "\n" |
|
412 |
|
. 'function rg_notify()' . "\n" |
|
413 |
|
. '{' . "\n" |
|
414 |
|
. ' echo "M${@}" > /dev/virtio-ports/rgw' . "\n" |
|
415 |
|
. '}' . "\n" |
|
416 |
|
. "\n" |
|
417 |
|
. 'rg_log_prefix=""' . "\n" |
|
418 |
|
. 'function xecho()' . "\n" |
|
419 |
|
. '{' . "\n" |
|
420 |
|
. ' echo "${rg_log_prefix}${@}"' . "\n" |
|
421 |
|
. '}' . "\n" |
|
422 |
|
. "\n" |
|
423 |
|
. 'function rg_log_push()' . "\n" |
|
424 |
|
. '{' . "\n" |
|
425 |
|
. ' [ "${1}" = "" ] || rg_notify "${1}"' . "\n" |
|
426 |
|
. ' [ "${1}" = "" ] || xecho "${1} [`date`]"' . "\n" |
|
427 |
|
. ' rg_log_prefix+=" "' . "\n" |
|
428 |
|
. '}' . "\n" |
|
429 |
|
. "\n" |
|
430 |
|
. 'function rg_log_pop()' . "\n" |
|
431 |
|
. '{' . "\n" |
|
432 |
|
. ' xecho "Done [E=${E}] [`date`]"' . "\n" |
|
433 |
|
. ' rg_log_prefix=${rg_log_prefix:2}' . "\n" |
|
434 |
|
. '}' . "\n" |
|
435 |
|
. "\n" |
|
436 |
|
. 'function rg_op_basic()' . "\n" |
|
437 |
|
. '{' . "\n" |
|
438 |
|
. ' E_CMD=${@}' . "\n" |
|
439 |
|
. ' xecho " X: ${E_CMD}" 1>&2' . "\n" // we do not want to interfere with user redirection (rg_op cmd > out) |
|
440 |
|
. ' "${@}" 2>~/rg_op.err' . "\n" |
|
441 |
|
. ' E=${?}' . "\n" |
|
442 |
|
. ' [ -r ~/rg_op.err ] \\' . "\n" |
|
443 |
|
. ' && sed -e "s|^|${rg_log_prefix} |" ~/rg_op.err 1>&2' . "\n" |
|
444 |
|
. ' if [ "${E}" != "0" ]; then' . "\n" |
|
445 |
|
. ' xecho " Cannot execute [${E_CMD}] [E=${E}]" 1>&2' . "\n" |
|
446 |
|
. ' fi' . "\n" |
|
447 |
|
. ' return ${E}' . "\n" |
|
448 |
|
. '}' . "\n" |
|
449 |
|
. "\n" |
|
450 |
|
. 'function rg_op()' . "\n" |
|
451 |
|
. '{' . "\n" |
|
452 |
|
. ' rg_op_basic "${@}"' . "\n" |
|
453 |
|
. ' return ${E}' . "\n" |
|
454 |
|
. '}' . "\n" |
|
455 |
|
. "\n" |
|
456 |
|
. '# No output redirect version - the output will not be captured by the caller' . "\n" |
|
457 |
|
. 'function rg_op_s()' . "\n" |
|
458 |
|
. '{' . "\n" |
|
459 |
|
. ' rg_op_basic "${@}" >~/rg_op.out' . "\n" |
|
460 |
|
. ' sed -e "s|^|${rg_log_prefix} |" ~/rg_op.out' . "\n" |
|
461 |
|
. ' return ${E}' . "\n" |
|
462 |
|
. '}' . "\n" |
|
463 |
|
. "\n" |
|
464 |
|
. 'function rg_cp()' . "\n" |
|
465 |
|
. '{' . "\n" |
|
466 |
|
. ' rg_op_s cp "${@}"' . "\n" |
|
467 |
|
. ' return ${E}' . "\n" |
|
468 |
|
. '}' . "\n" |
|
469 |
|
. "\n" |
|
470 |
|
. 'function do_sensitive_ops()' . "\n" |
|
471 |
|
. '{' . "\n" |
|
472 |
|
. ' [ "${job_uid}" = "0" ] && echo 1 && return' . "\n" |
|
473 |
|
. ' [ "${pkg_repo_uid}" = "${job_uid}" ] && echo 1 && return' . "\n" |
|
474 |
|
. ' echo 0' . "\n" |
|
475 |
|
. '}' . "\n" |
|
476 |
|
. "\n" |
|
477 |
|
. 'function rg_rpm_sign()' . "\n" |
|
478 |
|
. '{' . "\n" |
|
479 |
|
. ' local pkg_repo_id=${1}; shift' . "\n" |
|
480 |
|
. ' local dir=${1}; shift' . "\n" |
|
481 |
|
. ' local pkg_list_file=${1}; shift' . "\n" |
|
482 |
|
. "\n" |
|
483 |
|
. ' xecho "rg_rpm_sign: pkg_list_file=${pkg_list_file} key=repo_id-${pkg_repo_id} dir=${dir} [`date`]"' . "\n" |
|
484 |
|
. ' if [ ! -s "${pkg_list_file}" ]; then' . "\n" |
|
485 |
|
. ' xecho " No packages available."' . "\n" |
|
486 |
|
. ' return 0' . "\n" |
|
487 |
|
. ' fi' . "\n" |
|
488 |
|
. ' if [ ! -d "${dir}" ]; then' . "\n" |
|
489 |
|
. ' xecho " Dir ${dir} not found."' . "\n" |
|
490 |
|
. ' return 0' . "\n" |
|
491 |
|
. ' fi' . "\n" |
|
492 |
|
. "\n" |
|
493 |
|
. ' local old_dir=${PWD}' . "\n" |
|
494 |
|
. ' cd "${dir}"' . "\n" |
|
495 |
|
. ' rg_op_s rpmsign --define "_gpg_name repo_id-${pkg_repo_id}" \\' . "\n" |
|
496 |
|
. ' --define "_gpg_sign_cmd_extra_args --no-autostart --batch" \\' . "\n" |
|
497 |
|
. ' --resign $(< "${pkg_list_file}") </dev/null' . "\n" |
|
498 |
|
. ' cd "${old_dir}"' . "\n" |
|
499 |
|
. ' return ${E}' . "\n" |
|
500 |
|
. '}' . "\n" |
|
501 |
|
. "\n" |
|
502 |
|
. 'function rg_create_list()' . "\n" |
|
503 |
|
. '{' . "\n" |
|
504 |
|
. ' local dir=${1}; shift' . "\n" |
|
505 |
|
. ' local pkg_list_file=${1}; shift' . "\n" |
|
506 |
|
. "\n" |
|
507 |
|
. ' xecho "rg_create_list: pkg_list_file=${pkg_list_file} dir=${dir} [`date`]"' . "\n" |
|
508 |
|
. ' if [ ! -d "${dir}" ]; then' . "\n" |
|
509 |
|
. ' xecho " Dir ${dir} not found."' . "\n" |
|
510 |
|
. ' return 0' . "\n" |
|
511 |
|
. ' fi' . "\n" |
|
512 |
|
. ' export dir' . "\n" |
|
513 |
|
. ' (cd "${dir}"; find . -type f -iname "*.rpm") > "${pkg_list_file}"' . "\n" |
|
514 |
|
. ' if [ ! -s "${pkg_list_file}" ]; then' . "\n" |
|
515 |
|
. ' xecho " No files!"' . "\n" |
|
516 |
|
. ' return 0' . "\n" |
|
517 |
|
. ' fi' . "\n" |
|
518 |
|
. ' xecho " content: "`cat "${pkg_list_file}"`' . "\n" |
|
519 |
|
. '}' . "\n" |
|
520 |
|
. "\n" |
|
521 |
|
. 'function rg_rpm_createrepo()' . "\n" |
|
522 |
|
. '{' . "\n" |
|
523 |
|
. ' local distro=${1}; shift' . "\n" |
|
524 |
|
. ' local dir=${1}; shift' . "\n" |
|
525 |
|
. ' local pkg_list_file=${1}; shift' . "\n" |
|
526 |
|
. ' local oldpackagedirs=${1}; shift' . "\n" |
|
527 |
|
. "\n" |
|
528 |
|
. ' xecho "rg_rpm_createrepo dir=${dir} pkg_list_file=${pkg_list_file} oldpackagedirs=${oldpackagedirs} [`date`]"' . "\n" |
|
529 |
|
. ' rm -rf "${dir}/.repodata"' . "\n" |
|
530 |
|
. "\n" |
|
531 |
|
. ' local extra_paras=""' . "\n" |
|
532 |
|
. ' if [ "${env_distro}" = "fedora" ]; then' . "\n" |
|
533 |
|
. ' extra_paras=" --zck"' . "\n" |
|
534 |
|
. ' fi' . "\n" |
|
535 |
|
. "\n" |
|
536 |
|
. ' local old_dir="${PWD}"' . "\n" |
|
537 |
|
. ' cd "${dir}"' . "\n" |
|
538 |
|
. ' xecho " Switch to dir ${dir}"' . "\n" |
|
539 |
|
. ' xecho "DEBUG: Files:"' . "\n" |
|
540 |
|
. ' find .' . "\n" |
|
541 |
|
. ' rg_op_s createrepo -v --update ${extra_paras} \\' . "\n" |
|
542 |
|
. ' --recycle-pkglist \\' . "\n" |
|
543 |
|
. ' --retain-old-md 1 \\' . "\n" |
|
544 |
|
. ' --pkglist "${pkg_list_file}" \\' . "\n" |
|
545 |
|
. ' --local-sqlite \\' . "\n" |
|
546 |
|
. ' --deltas --num-deltas 2 \\'. "\n" |
|
547 |
|
. ' --oldpackagedirs . \\' . "\n" |
|
548 |
|
. ' --oldpackagedirs "${oldpackagedirs}" \\' . "\n" |
|
549 |
|
. ' .' . "\n" |
|
550 |
|
. ' cd "${old_dir}"' . "\n" |
|
551 |
|
. ' return ${E}' . "\n" |
|
552 |
|
. '}' . "\n" |
|
553 |
|
. "\n" |
|
554 |
|
. 'function rg_extract_rpm_info()' . "\n" |
|
555 |
|
. '{' . "\n" |
|
556 |
|
. ' local dir=${1}; shift' . "\n" |
|
557 |
|
. "\n" |
|
558 |
|
. ' xecho "rg_extract_rpm_info: dir=${dir} [`date`]"' . "\n" |
|
559 |
|
. ' spec_file=`find "${dir}" -type f -iname \'*.spec\' | head -n1`' . "\n" |
|
560 |
|
. ' if [ "${spec_file}" = "" ]; then' . "\n" |
|
561 |
|
. ' E=1' . "\n" |
|
562 |
|
. ' xecho " Cannot find any .spec file in ${dir}!" > ~/rg_op.err' . "\n" |
|
563 |
|
. ' return ${E}' . "\n" |
|
564 |
|
. ' fi' . "\n" |
|
565 |
|
. "\n" |
|
566 |
|
. ' spec_file_base=`basename "${spec_file}"`' . "\n" |
|
567 |
|
. ' xecho " .spec file: ${spec_file} basename=[${spec_file_base}]."' . "\n" |
|
568 |
|
. "\n" |
|
569 |
|
. ' xecho " Extracting info about version"' . "\n" |
|
570 |
|
. ' spec_name=`grep ^Name: "${spec_file}" | cut -d: -f2| tr -d " \t\r\n"`' . "\n" |
|
571 |
|
. ' spec_ver=`grep ^Version: "${spec_file}" | cut -d: -f2 | tr -d " \t\r\n"`' . "\n" |
|
572 |
|
. ' spec_rel=`grep ^Release: "${spec_file}" | cut -d: -f2 | tr -d " \t\r\n"`' . "\n" |
|
573 |
|
. ' spec_deps=`rpmspec --parse "${spec_file}" | grep ^BuildRequires | cut -d: -f2 | tr ",\t\r\n" " "`' . "\n" |
|
574 |
|
. ' xecho " DEBUG: name=${spec_name}, ver=${spec_ver}, rel=${spec_rel}, deps=[${spec_deps}]"' . "\n" |
|
575 |
|
. ' if [ "${spec_deps}" != "" ]; then' . "\n" |
|
576 |
|
. ' xecho " Storing deps [${spec_deps}] into /mnt/target/rocketgit/_deps"' . "\n" |
|
577 |
|
. ' echo "${spec_deps}" > /mnt/target/rocketgit/_deps' . "\n" |
|
578 |
|
. ' fi' . "\n" |
|
579 |
|
. ' echo' . "\n" |
|
580 |
|
. '}' . "\n" |
|
581 |
|
. "\n" |
|
582 |
|
. 'function rg_build_dotrepo()' . "\n" |
|
583 |
|
. '{' . "\n" |
|
584 |
|
. ' local dir=${1}; shift' . "\n" |
|
585 |
|
. "\n" |
|
586 |
|
. ' rg_log_push "rg_build_dotrepo dir=${dir} [`date`]"' . "\n" |
|
587 |
|
. ' while [ 1 ]; do' . "\n" |
|
588 |
|
. ' rg_extract_rpm_info "${dir}" || break' . "\n" |
|
589 |
|
. ' local dst_tar_gz="/mnt/rpmbuild/SOURCES/${spec_name}-${spec_ver}.tar.gz"' . "\n" |
|
590 |
|
. ' local dst_spec="/mnt/rpmbuild/SPECS/${spec_file_base}"' . "\n" |
|
591 |
|
. ' local xecho "DEBUG: dst_spec=${dst_spec}."' . "\n" |
|
592 |
|
. "\n" |
|
593 |
|
. ' xecho "Creating tar archive [${dst_tar_gz}]"' . "\n" |
|
594 |
|
. ' rg_op_s tar czf "${dst_tar_gz}" --transform "s:^:./${spec_name}-${spec_ver}/:g" \\' . "\n" |
|
595 |
|
. ' -C "${dir}" . || break' . "\n" |
|
596 |
|
. "\n" |
|
597 |
|
. ' xecho "Copying spec file [${spec_file}] -> [${dst_spec}]"' . "\n" |
|
598 |
|
. ' rg_cp "${spec_file}" "${dst_spec}"' . "\n" |
|
599 |
|
. "\n" |
|
600 |
|
. ' xecho "Making only binary packages"' . "\n" |
|
601 |
|
. ' rg_op_s rpmbuild --define "_topdir /mnt/rpmbuild" \\' . "\n" |
|
602 |
|
. ' -bb "${dst_spec}" || break' . "\n" |
|
603 |
|
. ' break' . "\n" |
|
604 |
|
. ' done' . "\n" |
|
605 |
|
. ' rg_log_pop' . "\n" |
|
606 |
|
. ' return ${E}' . "\n" |
|
607 |
|
. '}' . "\n" |
|
608 |
|
. "\n" |
|
609 |
|
. 'function rg_build_rpm()' . "\n" |
|
610 |
|
. '{' . "\n" |
|
611 |
|
. ' local dir=${1}; shift' . "\n" |
|
612 |
|
. "\n" |
|
613 |
|
. ' rg_log_push "rg_build_rpm: dir=${dir} [`date`]"' . "\n" |
|
614 |
|
. ' while [ 1 ]; do' . "\n" |
|
615 |
|
. ' rg_extract_rpm_info "${dir}" || break' . "\n" |
|
616 |
|
. ' local dst_tar_gz="/mnt/rpmbuild/SOURCES/${repo_username}+${spec_name}-${spec_ver}.tar.gz"' . "\n" |
|
617 |
|
. ' local dst_spec="/mnt/rpmbuild/SPECS/${repo_username}+${spec_file_base}"' . "\n" |
|
618 |
|
. "\n" |
|
619 |
|
. ' xecho "Creating archive from git repo [${dst_tar_gz}]"' . "\n" |
|
620 |
|
. ' rg_op_s git -C "${dir}" archive -v \\' . "\n" |
|
621 |
|
. ' --prefix "./${repo_username}+${spec_name}-${spec_ver}/" \\' . "\n" |
|
622 |
|
. ' -o "${dst_tar_gz}" rgw || break' . "\n" |
|
623 |
|
. "\n" |
|
624 |
|
. ' xecho "Processing spec file [${spec_file}] -> [${dst_spec}]"' . "\n" |
|
625 |
|
. ' rg_op sed -r -e "s|^Name:(\s*)(.*)(\s*)|Name:\1${repo_username}+\2\3|" \\' . "\n" |
|
626 |
|
. ' "${spec_file}" > "${dst_spec}" || break' . "\n" |
|
627 |
|
. "\n" |
|
628 |
|
. ' xecho "Making source and binary packages"' . "\n" |
|
629 |
|
. ' rg_op_s rpmbuild --define "_topdir /mnt/rpmbuild" \\' . "\n" |
|
630 |
|
. ' -ba "${dst_spec}" || break' . "\n" |
|
631 |
|
. ' break' . "\n" |
|
632 |
|
. ' done' . "\n" |
|
633 |
|
. ' rg_log_pop' . "\n" |
|
634 |
|
. ' return ${E}' . "\n" |
|
635 |
|
. '}' . "\n" |
|
636 |
|
. "\n" |
|
637 |
|
. 'rg_notify "Script ${0} started"' . "\n" |
|
638 |
|
. "\n"; |
|
639 |
|
|
|
640 |
|
$f = $job['main'] . '/root/build_tools.sh'; |
|
641 |
|
$r = @file_put_contents($f, $s); |
|
642 |
|
if ($r === FALSE) { |
|
643 |
|
$reason = 'cannot save ' . $f; |
|
644 |
|
$reason2 = 'cannot save ' . $f . ': ' . rg_php_err(); |
|
645 |
|
return FALSE; |
|
646 |
|
} |
|
647 |
|
|
|
648 |
|
$r = @chmod($f, 0755); |
|
649 |
|
if ($r === FALSE) { |
|
650 |
|
$reason = 'cannot chmod ' . $f; |
|
651 |
|
$reason2 = 'cannot chmod ' . $f . ': (' . rg_php_err() . ')'; |
|
652 |
|
return FALSE; |
|
653 |
|
} |
|
654 |
|
|
|
655 |
|
return TRUE; |
|
656 |
|
} |
|
657 |
|
|
|
658 |
|
/* |
|
659 |
|
* Building rpms script |
|
660 |
|
*/ |
|
661 |
|
function start_worker_build_rpms($job, $conf, &$reason, &$reason2) |
|
662 |
|
{ |
|
663 |
|
//rg_log_debug('job: ' . print_r($job, TRUE)); |
|
664 |
|
//rg_log_debug('conf: ' . print_r($conf, TRUE)); |
|
665 |
|
|
|
666 |
|
$env = $conf['env'][$job['env']]; |
|
667 |
|
|
|
668 |
|
$s = '' |
|
669 |
|
. '#!/bin/bash' . "\n" |
|
670 |
|
. "\n" |
|
671 |
|
. 'source /mnt/build_tools.sh' . "\n" |
|
672 |
|
. "\n" |
|
673 |
|
. 'env_distro=' . escapeshellarg($job['env_distro']) . "\n" |
|
674 |
|
. 'env_major=' . escapeshellarg($job['env_major']) . "\n" |
|
675 |
|
. 'env_arch=' . escapeshellarg($job['env_arch']) . "\n" |
|
676 |
|
. "\n" |
|
677 |
|
. 'mkdir -p /mnt/rpmbuild/{SOURCES,SPECS}' . "\n"; |
|
678 |
|
|
|
679 |
|
if (isset($job['repo_username'])) |
|
680 |
|
$s .= 'repo_username=' . escapeshellarg($job['repo_username']) . "\n"; |
|
681 |
|
|
|
682 |
|
$s .= '' |
|
683 |
|
. "\n" |
|
684 |
|
. 'while [ 1 ]; do' . "\n"; |
|
685 |
|
|
|
686 |
|
if (isset($job['url'])) |
|
687 |
|
$s .= ' rg_build_rpm /mnt/target || break' . "\n"; |
|
688 |
|
|
|
689 |
|
if (isset($job['rpm_repo_files'])) |
|
690 |
|
foreach ($job['rpm_repo_files'] as $pkg_subrepo_id => $junk) |
|
691 |
|
$s .= ' rg_build_dotrepo /mnt/target-dotrepo-' . $pkg_subrepo_id . ' || break' . "\n"; |
|
692 |
|
|
|
693 |
|
$s .= '' |
|
694 |
|
. ' break' . "\n" |
|
695 |
|
. 'done' . "\n"; |
|
696 |
|
|
|
697 |
|
$f = $job['main'] . '/root/build_rpms.sh'; |
|
698 |
|
$r = @file_put_contents($f, $s); |
|
699 |
|
if ($r === FALSE) { |
|
700 |
|
$reason = 'cannot save ' . $f; |
|
701 |
|
$reason2 = 'cannot save ' . $f . ': ' . rg_php_err(); |
|
702 |
|
return FALSE; |
|
703 |
|
} |
|
704 |
|
|
|
705 |
|
$r = @chmod($f, 0755); |
|
706 |
|
if ($r === FALSE) { |
|
707 |
|
$reason = 'cannot chmod ' . $f; |
|
708 |
|
$reason2 = 'cannot chmod ' . $f . ': (' . rg_php_err() . ')'; |
|
709 |
|
return FALSE; |
|
710 |
|
} |
|
711 |
|
|
|
712 |
|
return TRUE; |
|
713 |
|
} |
|
714 |
|
|
|
715 |
|
/* |
|
716 |
|
* Building rgfs building script |
|
717 |
|
* This is temporary, till we will have it on the main rocketgit repo. |
|
718 |
|
*/ |
|
719 |
|
function start_worker_build_rgfs($job, &$reason, &$reason2) |
|
720 |
|
{ |
|
721 |
|
$s = '' |
|
722 |
|
. '#!/bin/bash' . "\n" |
|
723 |
|
. "\n" |
|
724 |
|
. 'set -e' . "\n" |
|
725 |
|
. "\n" |
|
726 |
|
. 'if [ -x /usr/bin/rgfs ]; then' . "\n" |
|
727 |
|
. ' echo "rgfs binary already exists. Exit."' . "\n" |
|
728 |
|
. ' exit 0' . "\n" |
|
729 |
|
. 'fi' . "\n" |
|
730 |
|
. "\n" |
|
731 |
|
. 'git clone https://rocketgit.com/user/catalinux/rgfs' . "\n" |
|
732 |
|
. 'cd rgfs' . "\n" |
|
733 |
|
. './configure' . "\n" |
|
734 |
|
. 'make' . "\n" |
|
735 |
|
. 'make install' . "\n"; |
|
736 |
|
|
|
737 |
|
$f = $job['main'] . '/root/build_rgfs.sh'; |
|
738 |
|
$r = @file_put_contents($f, $s); |
|
739 |
|
if ($r === FALSE) { |
|
740 |
|
$reason = 'cannot save ' . $f; |
|
741 |
|
$reason2 = 'cannot save ' . $f . ': ' . rg_php_err(); |
|
742 |
|
return FALSE; |
|
743 |
|
} |
|
744 |
|
|
|
745 |
|
$r = @chmod($f, 0755); |
|
746 |
|
if ($r === FALSE) { |
|
747 |
|
$reason = 'cannot chmod ' . $f; |
|
748 |
|
$reason2 = 'cannot chmod ' . $f . ': (' . rg_php_err() . ')'; |
|
749 |
|
return FALSE; |
|
750 |
|
} |
|
751 |
|
|
|
752 |
|
return TRUE; |
|
753 |
|
} |
|
754 |
|
|
|
755 |
|
/* |
|
756 |
|
* Preparing packages for create repo phase |
|
757 |
|
* If allowed (pkg_repo[uid] == job[uid]), create repo. |
|
758 |
|
* TODO: we should move this script outside this code and pass only parameters. |
|
759 |
|
*/ |
|
760 |
|
function start_worker_build_repo($job, $conf, &$reason, &$reason2) |
|
761 |
|
{ |
|
762 |
|
if (empty($job['pkg_maps'])) { |
|
763 |
|
rg_log($job['id'] . ': DEBUG: no mappings => no repo will be build'); |
|
764 |
|
return TRUE; |
|
765 |
|
} |
|
766 |
|
|
|
767 |
|
$env = $conf['env'][$job['env']]; |
|
768 |
|
|
|
769 |
|
$b2 = $job['main'] . '/root/build2'; |
|
770 |
|
|
|
771 |
|
// Creating /mnt/build2 dir |
|
772 |
|
$r = @mkdir($b2, 0700); |
|
773 |
|
if ($r === FALSE) { |
|
774 |
|
$reason = 'cannot create main dir'; |
|
775 |
|
$reason2 = 'cannot create main dir (' . $job['main'] . '):' |
|
776 |
|
. ' (' . rg_php_err() . ')'; |
|
777 |
|
return FALSE; |
|
778 |
|
} |
|
779 |
|
|
|
780 |
|
// Save gpg keys |
|
781 |
|
$keys = 0; |
|
782 |
|
foreach ($job['pkg_repos'] as $pkg_repo_id => $info) { |
|
783 |
|
if ($info['uid'] != $job['uid']) { |
|
784 |
|
rg_log($job['id'] . ': DEBUG: skip saving key for' |
|
785 |
|
. ' pkg_repo_id ' . $pkg_repo_id |
|
786 |
|
. ' because the uids do not match'); |
|
787 |
|
continue; |
|
788 |
|
} |
|
789 |
|
|
|
790 |
|
// TODO: key was not generated - what should we do? |
|
791 |
|
if (empty($info['gpg_priv_key'])) |
|
792 |
|
continue; |
|
793 |
|
|
|
794 |
|
$r = @file_put_contents($b2 . '/gpg_priv_key_' . $pkg_repo_id . '.asc', |
|
795 |
|
$info['gpg_priv_key']); |
|
796 |
|
if ($r === FALSE) { |
|
797 |
|
$reason = 'cannot save gpg private key'; |
|
798 |
|
$reason2 = 'cannot save gpg private key: ' . rg_php_err(); |
|
799 |
|
return FALSE; |
|
800 |
|
} |
|
801 |
|
chmod($b2 . '/gpg_priv_key_' . $pkg_repo_id . '.asc', 0400); |
|
802 |
|
// No need for chown because we will make everything owned by build2 user. |
|
803 |
|
$keys++; |
|
804 |
|
} |
|
805 |
|
|
|
806 |
|
$s = '' |
|
807 |
|
. '#!/bin/bash' . "\n" |
|
808 |
|
. "\n" |
|
809 |
|
. 'source /mnt/build_tools.sh' . "\n"; |
|
810 |
|
|
|
811 |
|
$s .= "\n" |
|
812 |
|
. 'xecho "`date`"' . "\n" |
|
813 |
|
. 'xecho "I: job_id=${job_id} job_uid=${job_uid} repo_username=${repo_username}"' . "\n" |
|
814 |
|
. 'xecho "I: distro=${env_distro} major=${env_major} arch=${env_arch}"' . "\n" |
|
815 |
|
. "\n" |
|
816 |
|
. 'while [ 1 ]; do' |
|
817 |
|
. ' echo ' . "\n" |
|
818 |
|
. ' rg_log_push "Locating package files in /mnt/rpmbuild"' . "\n" |
|
819 |
|
. ' files=`find /mnt/rpmbuild -type f -iname "*.rpm" -o -iname "*.deb" | head -n1`' . "\n" |
|
820 |
|
. ' if [ "${files}" = "" ]; then' . "\n" |
|
821 |
|
. ' xecho "Not found."' . "\n" |
|
822 |
|
. ' else' . "\n" |
|
823 |
|
. ' xecho "Found ["${files}"]"' . "\n" |
|
824 |
|
. ' fi' . "\n" |
|
825 |
|
. ' rg_log_pop' . "\n" |
|
826 |
|
. "\n"; |
|
827 |
|
|
|
828 |
|
if ($keys > 0) |
|
829 |
|
$s .= '' |
|
830 |
|
. ' echo ' . "\n" |
|
831 |
|
. ' xecho "Importing ' . $keys . ' key(s)"' . "\n" |
|
832 |
|
. ' rg_op_s gpg --import /mnt/build2/gpg_priv_key_*.asc || break' . "\n" |
|
833 |
|
. ' # Signing something for test' . "\n" |
|
834 |
|
. ' rm -f /tmp/rg-sign-junk' . "\n" |
|
835 |
|
. ' rg_op_s gpg --output /tmp/rg-sign-junk --no-autostart --batch \\' . "\n" |
|
836 |
|
. ' --armour --sign /etc/hosts || break' . "\n"; |
|
837 |
|
|
|
838 |
|
$s .= "\n" |
|
839 |
|
. ' declare -A rgfs_mounted=()' . "\n"; |
|
840 |
|
|
|
841 |
|
if (isset($job['exec']['prepare_rpms'])) |
|
842 |
|
$s .= "\n" |
|
843 |
|
. ' echo' . "\n" |
|
844 |
|
. ' rg_log_push "== prepare rpms phase ====="' . "\n" |
|
845 |
|
. ' case "${env_distro}" in' . "\n" |
|
846 |
|
. ' fedora|centos)' . "\n" |
|
847 |
|
. ' for i in debug src os; do' . "\n" |
|
848 |
|
. ' src0="/mnt/rpmbuild"' . "\n" |
|
849 |
|
. ' src1="${src0}/RPMS"' . "\n" |
|
850 |
|
. ' src2="${src0}/SRPMS"' . "\n" |
|
851 |
|
. "\n" |
|
852 |
|
. ' rg_log_push "== ${i} ====="' . "\n" |
|
853 |
|
. ' for pkg_repo_id in ${pkg_repos_list}; do' . "\n" |
|
854 |
|
. ' pkg_repo_uid=${pri2uid[${pkg_repo_id}]}' . "\n" |
|
855 |
|
. ' rg_log_push "== pkg_repo_id ${pkg_repo_id} [uid ${pkg_repo_uid}] ====="' . "\n" |
|
856 |
|
. ' dir="/mnt/build2/rpms/${pkg_repo_id}/${i}"' . "\n" |
|
857 |
|
. ' if [ "`do_sensitive_ops`" = "1" ]; then' . "\n" |
|
858 |
|
. ' xecho "DEBUG: do_sensitive_ops==1: do not add username"' . "\n" |
|
859 |
|
. ' udir="${dir}"' . "\n" |
|
860 |
|
. ' else' . "\n" |
|
861 |
|
. ' xecho "DEBUG: do_sensitive_ops!=1: add username"' . "\n" |
|
862 |
|
. ' udir="${dir}/${repo_username}"' . "\n" |
|
863 |
|
. ' fi' . "\n" |
|
864 |
|
. "\n" |
|
865 |
|
. ' xecho "Copying RPMs to ${udir}"' . "\n" |
|
866 |
|
. ' rg_op_s mkdir -p "${udir}" || break' . "\n" |
|
867 |
|
. ' case ${i} in' . "\n" |
|
868 |
|
. ' debug)' . "\n" |
|
869 |
|
. ' if [ "`ls \"${src1}\"/*/*-debuginfo-*.rpm 2>/dev/null`" != "" ]; then' . "\n" |
|
870 |
|
. ' rg_cp -v --link "${src1}"/*/*-debuginfo-*.rpm "${udir}/"' . "\n" |
|
871 |
|
. ' fi' . "\n" |
|
872 |
|
. ' if [ "`ls \"${src1}\"/*/*-debugsource-*.rpm 2>/dev/null`" != "" ]; then' . "\n" |
|
873 |
|
. ' rg_cp -v --link "${src1}"/*/*-debugsource-*.rpm "${udir}/"' . "\n" |
|
874 |
|
. ' fi' . "\n" |
|
875 |
|
. ' ;;' . "\n" |
|
876 |
|
. ' src)' . "\n" |
|
877 |
|
. ' if [ "`ls \"${src2}\"/*.src.rpm 2>/dev/null`" != "" ]; then' . "\n" |
|
878 |
|
. ' rg_cp -v --link "${src2}"/*.src.rpm "${udir}/"' . "\n" |
|
879 |
|
. ' fi' . "\n" |
|
880 |
|
. ' ;;' . "\n" |
|
881 |
|
. ' os)' . "\n" |
|
882 |
|
. ' if [ "`ls \"${src1}\"/*/*.rpm 2>/dev/null`" != "" ]; then' . "\n" |
|
883 |
|
. ' rg_cp -v --link "${src1}"/*/*.rpm "${udir}/"' . "\n" |
|
884 |
|
. ' fi' . "\n" |
|
885 |
|
. ' ;;' . "\n" |
|
886 |
|
. ' esac' . "\n" |
|
887 |
|
. ' rg_log_pop' . "\n" |
|
888 |
|
. ' [ "${E}" = "0" ] || break' . "\n" |
|
889 |
|
. ' done' . "\n" |
|
890 |
|
. ' rg_log_pop' . "\n" |
|
891 |
|
. ' [ "${E}" = "0" ] || break' . "\n" |
|
892 |
|
. "\n" |
|
893 |
|
. ' # We must remove debug packages after copy, else we will copy them also in "os" dir' . "\n" |
|
894 |
|
. ' if [ "${i}" = "debug" ]; then' . "\n" |
|
895 |
|
. ' rm -f "${src1}"/*/*-debuginfo-*.rpm' . "\n" |
|
896 |
|
. ' rm -f "${src1}"/*/*-debugsource-*.rpm' . "\n" |
|
897 |
|
. ' fi' . "\n" |
|
898 |
|
. ' done' . "\n" |
|
899 |
|
. ' ;;' . "\n" |
|
900 |
|
. ' *) xecho "Unknown distro!"; E=1 ;;' . "\n" |
|
901 |
|
. ' esac' . "\n" |
|
902 |
|
. ' rg_log_pop' . "\n" |
|
903 |
|
. ' [ "${E}" = "0" ] || break' . "\n\n"; |
|
904 |
|
|
|
905 |
|
if (isset($job['exec']['prepare_rpms'])) |
|
906 |
|
foreach ($job['pkg_repos'] as $pkg_repo_id => $info) |
|
907 |
|
$s .= "\n" |
|
908 |
|
. ' pkg_repo_id=' . $pkg_repo_id . "\n" |
|
909 |
|
. ' pkg_repo_uid=' . $info['uid'] . "\n" |
|
910 |
|
. ' echo' . "\n" |
|
911 |
|
. ' rg_log_push "== create list and sign pkg_repo_id ${pkg_repo_id} ====="' . "\n" |
|
912 |
|
. ' case "${env_distro}" in' . "\n" |
|
913 |
|
. ' fedora|centos)' . "\n" |
|
914 |
|
. ' for i in debug src os; do' . "\n" |
|
915 |
|
. ' rg_log_push "== ${i} ====="' . "\n" |
|
916 |
|
. ' dir="/mnt/build2/rpms/${pkg_repo_id}/${i}"' . "\n" |
|
917 |
|
. ' pkg_list_file="${dir}/pkg.list"' . "\n" |
|
918 |
|
. "\n" |
|
919 |
|
. ' rg_create_list "${dir}" "${pkg_list_file}" || break' . "\n" |
|
920 |
|
. "\n" |
|
921 |
|
. ' if [ "`do_sensitive_ops`" != "1" ]; then' . "\n" |
|
922 |
|
. ' xecho "DEBUG: do_sensitive_ops!=1: skip rpm signing"' . "\n" |
|
923 |
|
. ' else' . "\n" |
|
924 |
|
. ' xecho "DEBUG: do_sensitive_ops==1: do signing"' . "\n" |
|
925 |
|
. ' rg_rpm_sign ${pkg_repo_id} "${dir}" "${pkg_list_file}" || break' . "\n" |
|
926 |
|
. ' fi' . "\n" |
|
927 |
|
. ' rg_log_pop' . "\n" |
|
928 |
|
. ' done' . "\n" |
|
929 |
|
. ' ;;' . "\n" |
|
930 |
|
. ' *) xecho "Unknown distro!"; E=1 ;;' . "\n" |
|
931 |
|
. ' esac' . "\n" |
|
932 |
|
. ' rg_log_pop' . "\n" |
|
933 |
|
. ' [ "${E}" = "0" ] || break' . "\n\n"; |
|
934 |
|
|
|
935 |
|
// TODO: we have a race for 'rg_op cat a > b': writing to b may fail => break is executed, but E is not set! |
|
936 |
|
foreach ($job['pkg_maps'] as $junk => $info) { |
|
937 |
|
$pkg_subrepo_id = $info['pkg_subrepo_id']; |
|
938 |
|
$si = $job['pkg_subrepos'][$pkg_subrepo_id]; |
|
939 |
|
$pkg_repo_id = $si['pkg_repo_id']; |
|
940 |
|
$ri = $job['pkg_repos'][$pkg_repo_id]; |
|
941 |
|
$s .= "\n" |
|
942 |
|
. ' export pkg_repo_id=' . $pkg_repo_id . "\n" |
|
943 |
|
. ' export pkg_repo_uid=' . $ri['uid'] . "\n" |
|
944 |
|
. ' export pkg_subrepo_id=' . $pkg_subrepo_id . "\n" |
|
945 |
|
. "\n" |
|
946 |
|
. ' echo' . "\n" |
|
947 |
|
. ' rg_log_push "== rgfs phase pkg_repo_id=${pkg_repo_id} pkg_subrepo_id=${pkg_subrepo_id} ====="' . "\n" |
|
948 |
|
. ' xecho "Mounting rgfs pkg_repo_id/pkg_subrepo_id=${pkg_repo_id}/${pkg_subrepo_id}"' . "\n" |
|
949 |
|
. ' export RGFS_LOG="/mnt/build2/rgfs/rgfs-${pkg_repo_id}-${pkg_subrepo_id}.log"' . "\n" |
|
950 |
|
. ' export RGFS_pkg_repo_id=${pkg_repo_id}' . "\n" |
|
951 |
|
. ' export RGFS_pkg_subrepo_id=${pkg_subrepo_id}' . "\n" |
|
952 |
|
. ' export RGFS_key=' . escapeshellarg($ri['rgfs_key']) . "\n" |
|
953 |
|
. ' dir="/mnt/build2/rgfs/${pkg_repo_id}/${pkg_subrepo_id}"' . "\n" |
|
954 |
|
. ' mkdir -p "${dir}"' . "\n" |
|
955 |
|
. ' # -s -> disable multithreading - not supported and not needed' . "\n" |
|
956 |
|
. ' # Please note, we cannot run strace on rgfs!' . "\n" |
|
957 |
|
. ' rg_op_s /usr/bin/rgfs -s -o kernel_cache -o auto_cache \\' . "\n" |
|
958 |
|
. ' -o noforget -o use_ino "${dir}" || break' . "\n" |
|
959 |
|
. ' rgfs_mounted[${pkg_repo_id}-${pkg_subrepo_id}]=1' . "\n" |
|
960 |
|
. "\n" |
|
961 |
|
. ' rgfs_subrepo_root="/mnt/build2/rgfs/${pkg_repo_id}/${pkg_subrepo_id}"' . "\n" |
|
962 |
|
. ' case "${env_distro}" in' . "\n" |
|
963 |
|
. ' fedora|centos)' . "\n" |
|
964 |
|
. ' for i in os debug src; do' . "\n" |
|
965 |
|
. ' rgfs="${rgfs_subrepo_root}/${env_distro}/${env_major}/${env_arch}/${i}"' . "\n" |
|
966 |
|
. ' rpms_dir="/mnt/build2/rpms/${pkg_repo_id}/${i}"' . "\n" |
|
967 |
|
. ' pkg_list_file="${rpms_dir}/pkg.list"' . "\n" |
|
968 |
|
. ' pkg_common_list_file="${rpms_dir}/pkg.common.list"' . "\n" |
|
969 |
|
. ' pkg_filter_list_file="${rpms_dir}/pkg.filter.list"' . "\n" |
|
970 |
|
. ' pkg_pend_list_file="/mnt/tmp/pkg.list"' . "\n" |
|
971 |
|
. ' > "${pkg_common_list_file}"' . "\n" |
|
972 |
|
. ' > "${pkg_pend_list_file}"' . "\n" |
|
973 |
|
. "\n" |
|
974 |
|
. ' rg_log_push "== ${i} ====="' . "\n" |
|
975 |
|
. ' mkdir -p "${rpms_dir}" # when we do not build packages, we may not have this dir' . "\n" |
|
976 |
|
. "\n" |
|
977 |
|
. ' rg_cp "${pkg_list_file}" "${pkg_common_list_file}" || break' . "\n" |
|
978 |
|
. ' if [ "${job_uid}" = "0" ]; then' . "\n" |
|
979 |
|
. ' if [ -d "${rgfs}/+pending" ]; then' . "\n" |
|
980 |
|
. ' rg_create_list "${rgfs}/+pending" "${pkg_pend_list_file}" || break' . "\n" |
|
981 |
|
. ' rg_op cat "${pkg_pend_list_file}" >> "${pkg_common_list_file}"' . "\n" |
|
982 |
|
. ' else' . "\n" |
|
983 |
|
. ' xecho "DEBUG: dir +pending does not exists."' . "\n" |
|
984 |
|
. ' fi' . "\n" |
|
985 |
|
. ' fi' . "\n" |
|
986 |
|
. ' xecho "DEBUG pkg_common_list_file: ["`cat "${pkg_common_list_file}"`"]"' . "\n" |
|
987 |
|
. "\n" |
|
988 |
|
. ' if [ ! -s "${pkg_common_list_file}" ]; then' . "\n" |
|
989 |
|
. ' xecho "pkg_common_list_files is empty"' . "\n" |
|
990 |
|
. ' rg_log_pop' . "\n" |
|
991 |
|
. ' continue' . "\n" |
|
992 |
|
. ' fi' . "\n" |
|
993 |
|
. "\n" |
|
994 |
|
. ' rg_log_push "Filtering existing packages from pkg_list_file"' . "\n" |
|
995 |
|
. ' > "${pkg_filter_list_file}"' . "\n" |
|
996 |
|
. ' while read file; do' . "\n" |
|
997 |
|
. ' if [ -r "${rgfs}/${file}" ]; then' . "\n" |
|
998 |
|
. ' xecho "- ${rgfs}/${file}"' . "\n" |
|
999 |
|
. ' continue' . "\n" |
|
1000 |
|
. ' fi' . "\n" |
|
1001 |
|
. ' xecho "+ ${rgfs}/${file}"' . "\n" |
|
1002 |
|
. ' echo "${file}" >> ${pkg_filter_list_file}' . "\n" |
|
1003 |
|
. ' done < "${pkg_common_list_file}"' . "\n" |
|
1004 |
|
. ' rg_log_pop' . "\n" |
|
1005 |
|
. "\n" |
|
1006 |
|
. ' if [ "${job_uid}" = "0" ]; then' . "\n" |
|
1007 |
|
. ' if [ -d "${rgfs}/+pending" ]; then' . "\n" |
|
1008 |
|
. ' xecho "+pending dir exists"' . "\n" |
|
1009 |
|
. ' rg_rpm_sign ${pkg_repo_id} "${rgfs}/+pending" "${pkg_pend_list_file}" || break' . "\n" |
|
1010 |
|
. ' for username in `(cd "${rgfs}/+pending"; ls)`; do' . "\n" |
|
1011 |
|
. ' mkdir -p "${rgfs}/${username}"' . "\n" |
|
1012 |
|
. ' rg_op_s mv -v "${rgfs}/+pending/${username}"/* "${rgfs}/${username}/" || break' . "\n" |
|
1013 |
|
. ' rg_op_s rmdir "${rgfs}/+pending/${username}" || break' . "\n" |
|
1014 |
|
. ' done' . "\n" |
|
1015 |
|
. ' [ "${E}" = "0" ] || break' . "\n" |
|
1016 |
|
. ' rg_op_s rmdir "${rgfs}/+pending" || break' . "\n" |
|
1017 |
|
. ' else' . "\n" |
|
1018 |
|
. ' xecho "+pending dir not found"' . "\n" |
|
1019 |
|
. ' fi' . "\n" |
|
1020 |
|
. ' fi' . "\n"; |
|
1021 |
|
|
|
1022 |
|
// TODO: job_uid -> repo_uid (everywhere)? |
|
1023 |
|
|
|
1024 |
|
if (isset($job['exec']['copy_to_rgfs'])) |
|
1025 |
|
$s .= "\n" |
|
1026 |
|
. ' xecho "exec:copy_to_rgfs"' . "\n" |
|
1027 |
|
. ' if [ -s "${pkg_list_file}" ]; then' . "\n" |
|
1028 |
|
. ' rg_log_push "Copying packages from ${rpms_dir} to ${rgfs}"' . "\n" |
|
1029 |
|
. ' rg_op_s mkdir -p "${rgfs}" || break' . "\n" |
|
1030 |
|
. ' if [ "${job_uid}" = "${pkg_repo_uid}" ]; then' . "\n" |
|
1031 |
|
. ' rg_cp -av "${rpms_dir}"/*.rpm "${rgfs}/" || break' . "\n" |
|
1032 |
|
. ' else' . "\n" |
|
1033 |
|
. ' # "+" is used because it is not allowed in username' . "\n" |
|
1034 |
|
. ' rg_op_s mkdir -p "${rgfs}/+pending/${repo_username}" || break' . "\n" |
|
1035 |
|
. ' rg_cp -av "${rpms_dir}/${repo_username}"/*.rpm "${rgfs}/+pending/${repo_username}/" || break' . "\n" |
|
1036 |
|
. ' fi' . "\n" |
|
1037 |
|
. ' rg_log_pop' . "\n" |
|
1038 |
|
. ' else' . "\n" |
|
1039 |
|
. ' xecho "No packages in pkg_list_file."' . "\n" |
|
1040 |
|
. ' fi'. "\n" |
|
1041 |
|
. "\n" |
|
1042 |
|
. ' if [ "`do_sensitive_ops`" != "1" ]; then' . "\n" |
|
1043 |
|
. ' xecho "Triggering the rebuild of subrepo"' . "\n" |
|
1044 |
|
. ' echo "${pkg_subrepo_id}" >> /mnt/build2/pkg_subrepo_dirty' . "\n" |
|
1045 |
|
. ' else' . "\n" |
|
1046 |
|
. ' rg_rpm_createrepo "${env_distro}" "${rgfs}" \\' . "\n" |
|
1047 |
|
. ' "${pkg_filter_list_file}" "${repo_username}" || break' . "\n" |
|
1048 |
|
. ' fi' . "\n"; |
|
1049 |
|
|
|
1050 |
|
$s .= "\n" |
|
1051 |
|
. ' rg_log_pop' . "\n" |
|
1052 |
|
. ' done ;;' . "\n" |
|
1053 |
|
. ' *) xecho "Unknown distro!"; E=1 ;;' . "\n" |
|
1054 |
|
. ' esac' . "\n" |
|
1055 |
|
. ' [ "${E}" = "0" ] || break' . "\n" |
|
1056 |
|
. "\n" |
|
1057 |
|
. ' if [ "${rgfs_mounted[${pkg_repo_id}-${pkg_subrepo_id}]}" = "1" ]; then' . "\n" |
|
1058 |
|
. ' xecho "Unmounting rgfs"' . "\n" |
|
1059 |
|
. ' umount "/mnt/build2/rgfs/${pkg_repo_id}/${pkg_subrepo_id}"' . "\n" |
|
1060 |
|
. ' fi' . "\n" |
|
1061 |
|
. ' rg_log_pop' . "\n"; |
|
1062 |
|
} |
|
1063 |
|
|
|
1064 |
|
$s .= "\n" |
|
1065 |
|
. ' xecho "Done [E=${E}] [`date`]"' . "\n" |
|
1066 |
|
. ' break' . "\n" |
|
1067 |
|
. 'done' . "\n" |
|
1068 |
|
. 'exit ${E}' . "\n"; |
|
1069 |
|
|
|
1070 |
|
$f = $job['main'] . '/root/build_repo.sh'; |
|
1071 |
|
$r = @file_put_contents($f, $s); |
|
1072 |
|
if ($r === FALSE) { |
|
1073 |
|
$reason = 'cannot save ' . $f; |
|
1074 |
|
$reason2 = 'cannot save ' . $f . ': ' . rg_php_err(); |
|
1075 |
|
return FALSE; |
|
1076 |
|
} |
|
1077 |
|
|
|
1078 |
|
$r = @chmod($f, 0755); |
|
1079 |
|
if ($r === FALSE) { |
|
1080 |
|
$reason = 'cannot chmod ' . $f; |
|
1081 |
|
$reason2 = 'cannot chmod ' . $f . ': (' . rg_php_err() . ')'; |
|
1082 |
|
return FALSE; |
|
1083 |
|
} |
|
1084 |
|
|
|
1085 |
|
return TRUE; |
|
1086 |
|
} |
|
1087 |
|
|
368 |
1088 |
/* |
/* |
369 |
1089 |
* Starts an worker |
* Starts an worker |
|
1090 |
|
* 'build' user is used for custom commands |
|
1091 |
|
* 'build2' user is used for packages |
370 |
1092 |
*/ |
*/ |
371 |
1093 |
function start_worker($job) |
function start_worker($job) |
372 |
1094 |
{ |
{ |
|
... |
... |
function start_worker($job) |
377 |
1099 |
rg_log_set_file($rg_log_dir . '/worker-' . $conf['id'] |
rg_log_set_file($rg_log_dir . '/worker-' . $conf['id'] |
378 |
1100 |
. '-' . $jid . '.log'); |
. '-' . $jid . '.log'); |
379 |
1101 |
|
|
380 |
|
rg_log('DEBUG: start_worker: job: ' . rg_array2string($job)); |
|
381 |
|
|
|
382 |
|
$env = $conf['env'][$job['env']]; |
|
383 |
|
rg_log_ml('DEBUG: start_worker: env: ' . print_r($env, TRUE)); |
|
384 |
|
|
|
385 |
|
$emain = escapeshellarg($job['main']); |
|
|
1102 |
|
rg_log_debug('start_worker: job: ' . rg_array2string($job)); |
386 |
1103 |
|
|
387 |
|
$name = 'rg-worker-' . $conf['id'] . '-' . $jid; |
|
388 |
|
$ename = escapeshellarg($name); |
|
389 |
|
$master = escapeshellarg($env['image']); |
|
390 |
|
$img = $job['main'] . '/image.qcow2'; // TODO: let admin control the image type |
|
391 |
|
$eimg = escapeshellarg($img); |
|
392 |
|
$img2 = $job['main'] . '/image2.raw'; |
|
393 |
|
$eimg2 = escapeshellarg($img2); |
|
|
1104 |
|
// Fake some properties (pkg repo rpm does not require them). |
|
1105 |
|
$list = array('username', 'head', 'repo_id'); |
|
1106 |
|
foreach ($list as $k) { |
|
1107 |
|
if (!isset($job[$k])) |
|
1108 |
|
$job[$k] = ''; |
|
1109 |
|
} |
394 |
1110 |
|
|
395 |
1111 |
$do_umount = FALSE; |
$do_umount = FALSE; |
396 |
1112 |
$err = TRUE; |
$err = TRUE; |
397 |
1113 |
$reason = ''; $reason2 = ''; |
$reason = ''; $reason2 = ''; |
398 |
1114 |
while (1) { |
while (1) { |
|
1115 |
|
$e = $job['env']; |
|
1116 |
|
if (!isset($conf['env'][$e])) { |
|
1117 |
|
$reason2 = 'cannot find environment ' . $e; |
|
1118 |
|
$reason = $reason2; |
|
1119 |
|
break; |
|
1120 |
|
} |
|
1121 |
|
|
|
1122 |
|
$env = $conf['env'][$e]; |
|
1123 |
|
rg_log_debug('start_worker: env info: ' . rg_array2string($env)); |
|
1124 |
|
|
|
1125 |
|
$emain = escapeshellarg($job['main']); |
|
1126 |
|
|
|
1127 |
|
$name = 'rg-worker-' . $conf['id'] . '-' . $jid; |
|
1128 |
|
$ename = escapeshellarg($name); |
|
1129 |
|
$master = escapeshellarg($env['image']); |
|
1130 |
|
$img = $job['main'] . '/image.qcow2'; // TODO: let admin control the image type |
|
1131 |
|
$eimg = escapeshellarg($img); |
|
1132 |
|
$img2 = $job['main'] . '/image2.raw'; |
|
1133 |
|
$eimg2 = escapeshellarg($img2); |
|
1134 |
|
|
399 |
1135 |
rg_exec('umount ' . $emain . '/root', '', FALSE, FALSE, FALSE); |
rg_exec('umount ' . $emain . '/root', '', FALSE, FALSE, FALSE); |
400 |
1136 |
|
|
401 |
1137 |
$r = rg_del_tree($job['main']); |
$r = rg_del_tree($job['main']); |
|
... |
... |
function start_worker($job) |
485 |
1221 |
break; |
break; |
486 |
1222 |
} |
} |
487 |
1223 |
|
|
488 |
|
// Clone repo |
|
489 |
|
$_env = 'GIT_SSH_COMMAND=ssh' |
|
490 |
|
. ' -o PasswordAuthentication=no' |
|
491 |
|
. ' -o ControlMaster=no' |
|
492 |
|
. ' -o IdentitiesOnly=yes' |
|
493 |
|
. ' -o IdentityFile=' . escapeshellarg($conf['state'] . '/key'); |
|
494 |
|
putenv($_env); |
|
495 |
|
$_s = time(); |
|
496 |
|
$cmd = 'git clone' |
|
497 |
|
. ' --recurse-submodules' |
|
498 |
|
//TODO . ' --shallow-submodules' |
|
499 |
|
. ' --no-checkout' |
|
500 |
|
. ' ' . escapeshellarg($job['url']) |
|
501 |
|
. ' ' . $emain . '/root/git'; |
|
502 |
|
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
|
503 |
|
@file_put_contents($job['main'] . '/root/T_clone', time() - $_s); |
|
504 |
|
if ($r['ok'] !== 1) { |
|
505 |
|
$reason = 'git clone error; contact admin'; |
|
506 |
|
$reason2 = $r['errmsg'] . ': ' . $r['stderr']; |
|
507 |
|
break; |
|
|
1224 |
|
$out_dir = $job['main'] . '/root/target'; |
|
1225 |
|
|
|
1226 |
|
if (isset($job['rpm_repo_files'])) { |
|
1227 |
|
$err = FALSE; |
|
1228 |
|
foreach ($job['rpm_repo_files'] as $pkg_subrepo_id => $files) { |
|
1229 |
|
rg_log_debug('Storing dotrepo files for subrepo ' . $pkg_subrepo_id); |
|
1230 |
|
$_out_dir = $out_dir . '-dotrepo-' . $pkg_subrepo_id; |
|
1231 |
|
foreach ($files as $f => $c) { |
|
1232 |
|
$r = rg_create_dirs($_out_dir . '/' . $f); |
|
1233 |
|
if ($r['ok'] != 1) { |
|
1234 |
|
$reason = 'cannot create dir'; |
|
1235 |
|
$reason2 = 'cannot create dir: ' . $r['errmsg']; |
|
1236 |
|
$err = TRUE; |
|
1237 |
|
break; |
|
1238 |
|
} |
|
1239 |
|
|
|
1240 |
|
$r = @file_put_contents($_out_dir . '/' . $f, $c); |
|
1241 |
|
if ($r === FALSE) { |
|
1242 |
|
$reason = 'cannot store rpm file'; |
|
1243 |
|
$reason2 = 'cannot store file: ' . rg_php_err(); |
|
1244 |
|
$err = TRUE; |
|
1245 |
|
break; |
|
1246 |
|
} |
|
1247 |
|
} |
|
1248 |
|
if ($err) |
|
1249 |
|
break; |
|
1250 |
|
} |
|
1251 |
|
if ($err) |
|
1252 |
|
break; |
|
1253 |
|
} |
|
1254 |
|
|
|
1255 |
|
if (isset($job['url'])) { |
|
1256 |
|
// Clone repo |
|
1257 |
|
$_env = 'GIT_SSH_COMMAND=ssh' |
|
1258 |
|
. ' -o PasswordAuthentication=no' |
|
1259 |
|
. ' -o ControlMaster=no' |
|
1260 |
|
. ' -o IdentitiesOnly=yes' |
|
1261 |
|
. ' -o IdentityFile=' . escapeshellarg($conf['state'] . '/key'); |
|
1262 |
|
putenv($_env); |
|
1263 |
|
$_s = time(); |
|
1264 |
|
$cmd = 'git clone' |
|
1265 |
|
. ' --recurse-submodules' |
|
1266 |
|
//TODO . ' --shallow-submodules' |
|
1267 |
|
. ' --no-checkout' |
|
1268 |
|
. ' ' . escapeshellarg($job['url']) |
|
1269 |
|
. ' ' . $emain . '/root/target'; |
|
1270 |
|
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
|
1271 |
|
rg_log_debug('clone: ' . print_r($r, TRUE)); |
|
1272 |
|
@file_put_contents($job['main'] . '/root/T_clone', time() - $_s); |
|
1273 |
|
if ($r['ok'] !== 1) { |
|
1274 |
|
$reason = 'git clone error; contact admin'; |
|
1275 |
|
$reason2 = $r['errmsg'] . ': ' . $r['stderr']; |
|
1276 |
|
break; |
|
1277 |
|
} |
508 |
1278 |
} |
} |
509 |
1279 |
|
|
510 |
1280 |
// Creating build.sh file |
// Creating build.sh file |
|
... |
... |
function start_worker($job) |
515 |
1285 |
$s .= "\n"; |
$s .= "\n"; |
516 |
1286 |
|
|
517 |
1287 |
// build.sh: secrets |
// build.sh: secrets |
518 |
|
if (isset($job['secrets'])) { |
|
|
1288 |
|
if (isset($job['secrets']) && !empty($job['secrets'])) { |
519 |
1289 |
$s .= '# Secrets' . "\n"; |
$s .= '# Secrets' . "\n"; |
520 |
1290 |
foreach ($job['secrets'] as $i => $info) { |
foreach ($job['secrets'] as $i => $info) { |
521 |
1291 |
if (empty($info['name'])) |
if (empty($info['name'])) |
|
... |
... |
function start_worker($job) |
526 |
1296 |
$s .= "\n"; |
$s .= "\n"; |
527 |
1297 |
} |
} |
528 |
1298 |
|
|
529 |
|
// build.sh: checkout |
|
530 |
|
$s .= 'cd /mnt/git' . "\n\n"; |
|
531 |
|
$s .= 'git branch -f rgw ' . escapeshellarg($job['head']) . " &>/mnt/status/git.branch.log\n\n"; |
|
532 |
|
$s .= 'git checkout rgw' . " &>/mnt/status/git.checkout.log\n\n"; |
|
|
1299 |
|
$s .= 'cd /mnt/target' . "\n\n"; |
|
1300 |
|
|
|
1301 |
|
if (isset($job['url'])) { |
|
1302 |
|
$s .= 'git branch -f rgw ' . escapeshellarg($job['head']) . " &>/mnt/status/git.branch.log\n\n"; |
|
1303 |
|
$s .= 'git checkout rgw' . " &>/mnt/status/git.checkout.log\n\n"; |
|
1304 |
|
} |
|
1305 |
|
|
|
1306 |
|
if (!isset($job['cmds'])) |
|
1307 |
|
$job['cmds'] = array(); |
533 |
1308 |
foreach ($job['cmds'] as $_name => $i) { |
foreach ($job['cmds'] as $_name => $i) { |
534 |
1309 |
if (empty($i['cmd'])) |
if (empty($i['cmd'])) |
535 |
1310 |
continue; |
continue; |
|
... |
... |
function start_worker($job) |
564 |
1339 |
. $lok |
. $lok |
565 |
1340 |
. 'fi' . "\n\n"; |
. 'fi' . "\n\n"; |
566 |
1341 |
} |
} |
567 |
|
//rg_log_ml('DEBUG: build.sh: ' . $s); |
|
|
1342 |
|
//rg_log_debug('build.sh: ' . $s); |
568 |
1343 |
$r = @file_put_contents($job['main'] . '/root/build.sh', $s); |
$r = @file_put_contents($job['main'] . '/root/build.sh', $s); |
569 |
1344 |
if ($r === FALSE) { |
if ($r === FALSE) { |
570 |
1345 |
$reason = 'cannot store build commands'; |
$reason = 'cannot store build commands'; |
|
... |
... |
function start_worker($job) |
582 |
1357 |
// on a single line to avoid error if one is not available |
// on a single line to avoid error if one is not available |
583 |
1358 |
$p_i_cmd = ''; |
$p_i_cmd = ''; |
584 |
1359 |
if (!empty($job['packages'])) { |
if (!empty($job['packages'])) { |
585 |
|
rg_log('DEBUG: packages: ' . $job['packages'] . '.'); |
|
|
1360 |
|
rg_log_debug('packages: ' . $job['packages'] . '.'); |
586 |
1361 |
$pkgs = explode(' ', $job['packages']); |
$pkgs = explode(' ', $job['packages']); |
587 |
1362 |
$p_i_cmd .= '> /mnt/packages.log' . "\n"; |
$p_i_cmd .= '> /mnt/packages.log' . "\n"; |
588 |
1363 |
|
|
|
... |
... |
function start_worker($job) |
591 |
1366 |
$p_i_cmd .= 'export APT_LISTCHANGES_FRONTEND=none' . "\n"; |
$p_i_cmd .= 'export APT_LISTCHANGES_FRONTEND=none' . "\n"; |
592 |
1367 |
|
|
593 |
1368 |
foreach ($pkgs as $p) { |
foreach ($pkgs as $p) { |
594 |
|
$p_i_cmd .= $env['pkg_cmd'] |
|
595 |
|
. ' ' . escapeshellarg($p) |
|
596 |
|
. ' >> /mnt/packages.log 2>&1' . "\n"; |
|
|
1369 |
|
$p_i_cmd .= '(' . "\n" |
|
1370 |
|
. 'echo "=== ' . $p . ' ==="' . "\n" |
|
1371 |
|
. $env['pkg_cmd'] |
|
1372 |
|
. ' ' . escapeshellarg($p) . "\n" |
|
1373 |
|
. 'echo; echo' . "\n" |
|
1374 |
|
. ') >> /mnt/packages.log 2>&1' . "\n"; |
597 |
1375 |
} |
} |
598 |
1376 |
} |
} |
599 |
1377 |
|
|
|
1378 |
|
$r = start_worker_build_tools($job, $conf, $reason, $reason2); |
|
1379 |
|
if ($r === FALSE) |
|
1380 |
|
break; |
|
1381 |
|
|
|
1382 |
|
$r = start_worker_build_rpms($job, $conf, $reason, $reason2); |
|
1383 |
|
if ($r === FALSE) |
|
1384 |
|
break; |
|
1385 |
|
|
|
1386 |
|
$r = start_worker_build_rgfs($job, $reason, $reason2); |
|
1387 |
|
if ($r === FALSE) |
|
1388 |
|
break; |
|
1389 |
|
|
|
1390 |
|
$r = start_worker_build_repo($job, $conf, $reason, $reason2); |
|
1391 |
|
if ($r === FALSE) |
|
1392 |
|
break; |
|
1393 |
|
|
600 |
1394 |
// Store commands in rg.sh (executed at boot) |
// Store commands in rg.sh (executed at boot) |
601 |
1395 |
// TODO: we should not wait for net forever |
// TODO: we should not wait for net forever |
602 |
1396 |
$r = @file_put_contents($job['main'] . '/root/rg.sh', |
$r = @file_put_contents($job['main'] . '/root/rg.sh', |
603 |
1397 |
'#!/bin/bash' . "\n" |
'#!/bin/bash' . "\n" |
|
1398 |
|
. 'echo "This is the rg.sh script..."' . "\n" |
|
1399 |
|
. "\n" |
604 |
1400 |
. 'mkdir /mnt/tmp && chmod 1777 /mnt/tmp' . "\n" |
. 'mkdir /mnt/tmp && chmod 1777 /mnt/tmp' . "\n" |
605 |
1401 |
. 'mount --bind /mnt/tmp /tmp' . "\n" |
. 'mount --bind /mnt/tmp /tmp' . "\n" |
606 |
1402 |
. 'mkdir -p /mnt/var/tmp && chmod 1777 /mnt/var/tmp' . "\n" |
. 'mkdir -p /mnt/var/tmp && chmod 1777 /mnt/var/tmp' . "\n" |
607 |
1403 |
. 'mount --bind /mnt/var/tmp /var/tmp' . "\n" |
. 'mount --bind /mnt/var/tmp /var/tmp' . "\n" |
608 |
1404 |
. "\n" |
. "\n" |
609 |
|
. 'mkdir /mnt/status' . "\n" |
|
610 |
|
. 'chown -R build:build /mnt/git /mnt/status' . "\n" |
|
|
1405 |
|
. 'mkdir -p /mnt/target /mnt/rpmbuild /mnt/status' . "\n" |
|
1406 |
|
. 'chown -R build:build /mnt/target* /mnt/rpmbuild /mnt/status' . "\n" |
|
1407 |
|
. 'mkdir -p /mnt/build2' . "\n" |
|
1408 |
|
. 'chown -R build2:build2 /mnt/build2; chmod go= /mnt/build2' . "\n" |
611 |
1409 |
. 'echo "PATH=${PATH}"' . "\n" |
. 'echo "PATH=${PATH}"' . "\n" |
612 |
1410 |
. 'ERR=""' . "\n" |
. 'ERR=""' . "\n" |
613 |
1411 |
. 'id' . "\n" |
. 'id' . "\n" |
614 |
1412 |
. 'date +%s > /mnt/T_START' . "\n" |
. 'date +%s > /mnt/T_START' . "\n" |
615 |
|
. '# Waiting for net...' . "\n" |
|
|
1413 |
|
. 'chmod a+rw /dev/virtio-ports/rgw # Else, we cannot signal from inside the build' . "\n" |
|
1414 |
|
. 'echo "MVM started" > /dev/virtio-ports/rgw' . "\n" |
|
1415 |
|
. "\n" |
|
1416 |
|
. 'echo "MWaiting for net" > /dev/virtio-ports/rgw' . "\n" |
|
1417 |
|
. '# Waiting for net' . "\n" |
616 |
1418 |
. 'while [ "`ip ro li | grep ^default`" = "" ]; do' . "\n" |
. 'while [ "`ip ro li | grep ^default`" = "" ]; do' . "\n" |
617 |
1419 |
. ' sleep 1' . "\n" |
. ' sleep 1' . "\n" |
618 |
1420 |
. 'done' . "\n" |
. 'done' . "\n" |
619 |
1421 |
. 'date +%s > /mnt/T_NET_OK' . "\n\n" |
. 'date +%s > /mnt/T_NET_OK' . "\n\n" |
620 |
1422 |
. "\n" |
. "\n" |
|
1423 |
|
. 'echo "MInstalling packages" > /dev/virtio-ports/rgw' . "\n" |
621 |
1424 |
. $p_i_cmd |
. $p_i_cmd |
622 |
1425 |
. 'date +%s > /mnt/T_PKGS_OK' . "\n\n" |
. 'date +%s > /mnt/T_PKGS_OK' . "\n\n" |
623 |
1426 |
. "\n" |
. "\n" |
|
1427 |
|
. 'modprobe fuse' . "\n" |
|
1428 |
|
. "\n" |
624 |
1429 |
. '# Disabling further module loading.' . "\n" |
. '# Disabling further module loading.' . "\n" |
625 |
1430 |
. 'echo 1 > /proc/sys/kernel/modules_disabled' . "\n" |
. 'echo 1 > /proc/sys/kernel/modules_disabled' . "\n" |
626 |
1431 |
. "\n" |
. "\n" |
|
... |
... |
function start_worker($job) |
643 |
1448 |
. ' fi' . "\n" |
. ' fi' . "\n" |
644 |
1449 |
. 'fi' . "\n" |
. 'fi' . "\n" |
645 |
1450 |
. "\n" |
. "\n" |
646 |
|
. 'if [ "${ERR}" = "" ]; then' . "\n" |
|
|
1451 |
|
. 'while [ "${ERR}" = "" ]; do' . "\n" |
647 |
1452 |
. ' su - build -c "bash /mnt/build.sh" &>/mnt/status/build.log' . "\n" |
. ' su - build -c "bash /mnt/build.sh" &>/mnt/status/build.log' . "\n" |
648 |
|
. ' sync' . "\n" |
|
649 |
|
. 'else' . "\n" |
|
|
1453 |
|
. ' date +%s > /mnt/T_RPMS_START' . "\n" |
|
1454 |
|
. ' su - build -c "bash /mnt/build_rpms.sh" &>/mnt/status/build_rpms.log' . "\n" |
|
1455 |
|
. ' E=${?}' . "\n" |
|
1456 |
|
. ' date +%s > /mnt/T_RPMS_END' . "\n" |
|
1457 |
|
. ' if [ "${E}" != "0" ]; then' . "\n" |
|
1458 |
|
. ' ERR=$(< ~/rg_op.err)' . "\n" |
|
1459 |
|
. ' break' . "\n" |
|
1460 |
|
. ' fi' . "\n" |
|
1461 |
|
. "\n" |
|
1462 |
|
. ' _deps=`cat /mnt/target/rocketgit/_deps 2>/dev/null`' . "\n" |
|
1463 |
|
. ' if [ "${_deps}" != "" ]; then' . "\n" |
|
1464 |
|
. ' echo; echo "`date`: Installing dependencies [${_deps}]"' . "\n" |
|
1465 |
|
. ' export DEBIAN_FRONTEND=noninteractive' . "\n" |
|
1466 |
|
. ' export APT_LISTCHANGES_FRONTEND=none' . "\n" |
|
1467 |
|
. ' ' . $env['pkg_cmd'] . ' ${_deps}' . "\n" |
|
1468 |
|
. ' if [ "${?}" != "0" ]; then' . "\n" |
|
1469 |
|
. ' ERR="cannot install deps"' . "\n" |
|
1470 |
|
. ' break' . "\n" |
|
1471 |
|
. ' fi' . "\n" |
|
1472 |
|
. ' fi' . "\n" |
|
1473 |
|
. "\n" |
|
1474 |
|
. ' rm -f ~/rg_op.err # it was owned by build user till now' . "\n" |
|
1475 |
|
. ' # Moving everything to build2 dir. TODO: Why?' . "\n" |
|
1476 |
|
. ' mv /mnt/target/rocketgit /mnt/build2' . "\n" |
|
1477 |
|
. ' chown -R build2:build2 /mnt/build2 /mnt/rpmbuild' . "\n" |
|
1478 |
|
. "\n" |
|
1479 |
|
. ' echo; echo "`date`: Running build_rgfs.sh script"' . "\n" |
|
1480 |
|
. ' bash /mnt/build_rgfs.sh &>/mnt/status/build_rgfs.log' . "\n" |
|
1481 |
|
. ' if [ "${?}" != "0" ]; then' . "\n" |
|
1482 |
|
. ' echo "error building rgfs; trying to continue"' . "\n" |
|
1483 |
|
. ' fi' . "\n" |
|
1484 |
|
. "\n" |
|
1485 |
|
. ' if [ -r /mnt/build_repo.sh ]; then' . "\n" |
|
1486 |
|
. ' echo; echo "`date`: Running build_repo.sh script"' . "\n" |
|
1487 |
|
. ' date +%s > /mnt/T_REPO_START' . "\n" |
|
1488 |
|
. ' su - build2 -c "bash /mnt/build_repo.sh" &>/mnt/status/build_repo.log' . "\n" |
|
1489 |
|
. ' E=${?}' . "\n" |
|
1490 |
|
. ' date +%s > /mnt/T_REPO_END' . "\n" |
|
1491 |
|
. ' if [ "${E}" != "0" ]; then' . "\n" |
|
1492 |
|
. ' ERR=$(< ~/rg_op.err)' . "\n" |
|
1493 |
|
. ' break' . "\n" |
|
1494 |
|
. ' fi' . "\n" |
|
1495 |
|
. ' fi' . "\n" |
|
1496 |
|
. "\n" |
|
1497 |
|
. ' break' . "\n" |
|
1498 |
|
. 'done' . "\n" |
|
1499 |
|
. 'if [ "${ERR}" != "" ]; then' . "\n" |
650 |
1500 |
. ' echo "${ERR}" > /mnt/status/err' . "\n" |
. ' echo "${ERR}" > /mnt/status/err' . "\n" |
|
1501 |
|
. ' echo "EERROR: ${ERR}" > /dev/virtio-ports/rgw' . "\n" |
651 |
1502 |
. 'fi' . "\n" |
. 'fi' . "\n" |
652 |
1503 |
. 'date +%s > /mnt/T_DONE' . "\n\n" |
. 'date +%s > /mnt/T_DONE' . "\n\n" |
653 |
|
. 'shutdown -h now' |
|
|
1504 |
|
. 'echo "MDONE" > /dev/virtio-ports/rgw' . "\n" |
|
1505 |
|
. "\n" |
|
1506 |
|
. 'echo "`date`: Syncing"' . "\n" |
|
1507 |
|
. 'sync' . "\n" |
|
1508 |
|
. "\n" |
|
1509 |
|
. 'echo "`date`: Shutting down"' . "\n" |
|
1510 |
|
. 'while [ -r /root/stop ]; do' . "\n" |
|
1511 |
|
. ' sleep 1' . "\n" |
|
1512 |
|
. 'done' . "\n" |
|
1513 |
|
. 'poweroff -f' . "\n" |
654 |
1514 |
); |
); |
655 |
1515 |
if ($r === FALSE) { |
if ($r === FALSE) { |
656 |
1516 |
$reason = 'cannot store commands'; |
$reason = 'cannot store commands'; |
|
... |
... |
function start_worker($job) |
719 |
1579 |
rg_log('error2: ' . $reason2); |
rg_log('error2: ' . $reason2); |
720 |
1580 |
@file_put_contents($job['main'] . '/error.log', $reason); |
@file_put_contents($job['main'] . '/error.log', $reason); |
721 |
1581 |
@file_put_contents($job['main'] . '/error2.log', $reason2); |
@file_put_contents($job['main'] . '/error2.log', $reason2); |
722 |
|
} else { |
|
723 |
|
@file_put_contents($job['main'] . '/helper-ok.log', ''); |
|
|
1582 |
|
exit(1); |
724 |
1583 |
} |
} |
725 |
1584 |
|
|
726 |
1585 |
rg_log('Done'); |
rg_log('Done'); |
|
1586 |
|
exit(0); |
727 |
1587 |
} |
} |
728 |
1588 |
|
|
729 |
1589 |
/* |
/* |
|
... |
... |
function xhandle_one($key, $data) |
779 |
1639 |
return; |
return; |
780 |
1640 |
} |
} |
781 |
1641 |
|
|
|
1642 |
|
// some small fixes |
|
1643 |
|
$_x = explode('-', $u['env']); |
|
1644 |
|
$u['env_distro'] = $_x[0]; |
|
1645 |
|
$u['env_major'] = $_x[1]; |
|
1646 |
|
$u['env_arch'] = $_x[2]; |
|
1647 |
|
|
782 |
1648 |
$jobs[$jid] = $u; |
$jobs[$jid] = $u; |
783 |
1649 |
$jobs[$jid]['received'] = time(); |
$jobs[$jid]['received'] = time(); |
784 |
1650 |
$jobs[$jid]['done_last_sent'] = 0; |
$jobs[$jid]['done_last_sent'] = 0; |
785 |
1651 |
$jobs[$jid]['main'] = $conf['state'] . '/rocketgit-j-' . $jid; |
$jobs[$jid]['main'] = $conf['state'] . '/rocketgit-j-' . $jid; |
|
1652 |
|
$jobs[$jid]['pkg_subrepos_dirty'] = array(); |
|
1653 |
|
|
786 |
1654 |
// TODO: add bellow configuration to the web form |
// TODO: add bellow configuration to the web form |
787 |
1655 |
if (!isset($jobs[$jid]['disk_size_gib'])) |
if (!isset($jobs[$jid]['disk_size_gib'])) |
788 |
1656 |
$jobs[$jid]['disk_size_gib'] = '20'; |
$jobs[$jid]['disk_size_gib'] = '20'; |
789 |
1657 |
if (!isset($jobs[$jid]['mem_mib'])) |
if (!isset($jobs[$jid]['mem_mib'])) |
790 |
|
$jobs[$jid]['mem_mib'] = '500'; |
|
|
1658 |
|
$jobs[$jid]['mem_mib'] = '600'; |
791 |
1659 |
if (!isset($jobs[$jid]['cpus'])) |
if (!isset($jobs[$jid]['cpus'])) |
792 |
|
$jobs[$jid]['cpus'] = '1'; |
|
|
1660 |
|
$jobs[$jid]['cpus'] = '2'; |
793 |
1661 |
$jobs[$jid]['state'] = RG_JOB_INIT; |
$jobs[$jid]['state'] = RG_JOB_INIT; |
794 |
1662 |
$jobs[$jid]['dirty'] = 1; |
$jobs[$jid]['dirty'] = 1; |
795 |
1663 |
return; |
return; |
|
... |
... |
function xhandle($key, $data) |
877 |
1745 |
return $ret; |
return $ret; |
878 |
1746 |
} |
} |
879 |
1747 |
|
|
|
1748 |
|
/* |
|
1749 |
|
* Handle received commands by channel from VM |
|
1750 |
|
*/ |
|
1751 |
|
function xchan($key, $data) |
|
1752 |
|
{ |
|
1753 |
|
global $rg_conns; |
|
1754 |
|
global $features; |
|
1755 |
|
global $jobs; |
|
1756 |
|
|
|
1757 |
|
$jid = $rg_conns[$key]['jid']; |
|
1758 |
|
|
|
1759 |
|
$ret = 0; |
|
1760 |
|
while (1) { |
|
1761 |
|
$pos = strpos($data, "\n"); |
|
1762 |
|
if ($pos === FALSE) |
|
1763 |
|
return $ret; |
|
1764 |
|
|
|
1765 |
|
$one = substr($data, 0, $pos); |
|
1766 |
|
rg_log($key . ': DEBUG: received data from VM: ' . $one); |
|
1767 |
|
$data = substr($data, $pos + 1); |
|
1768 |
|
$ret += $pos + 1; |
|
1769 |
|
|
|
1770 |
|
if (!isset($features['notify'])) |
|
1771 |
|
continue; |
|
1772 |
|
|
|
1773 |
|
$j = array( |
|
1774 |
|
'op' => 'rg_notify', |
|
1775 |
|
'id' => $jid, |
|
1776 |
|
'm' => $one |
|
1777 |
|
); |
|
1778 |
|
rg_conn_enq('master', @json_encode($j) . "\n"); |
|
1779 |
|
} |
|
1780 |
|
|
|
1781 |
|
return $ret; |
|
1782 |
|
} |
|
1783 |
|
|
|
1784 |
|
/* |
|
1785 |
|
* Tries to open a communication channel with the VM for real-time communication |
|
1786 |
|
*/ |
|
1787 |
|
function xchan_open(&$job) |
|
1788 |
|
{ |
|
1789 |
|
global $rg_conns; |
|
1790 |
|
|
|
1791 |
|
// We are trying to open the communication channel with the guest |
|
1792 |
|
do { |
|
1793 |
|
$jid = $job['id']; |
|
1794 |
|
|
|
1795 |
|
$s = @socket_create(AF_UNIX, SOCK_STREAM, 0); |
|
1796 |
|
if ($s === FALSE) { |
|
1797 |
|
rg_log($jid . ': Cannot create socket for channel comm:' |
|
1798 |
|
. ' ' . socket_strerror(socket_last_error())); |
|
1799 |
|
break; |
|
1800 |
|
} |
|
1801 |
|
|
|
1802 |
|
$r = @socket_connect($s, $job['main'] . '/x.chan'); |
|
1803 |
|
if ($r === FALSE) { |
|
1804 |
|
rg_log($jid . ': Cannot connect to sock for channel comm:' |
|
1805 |
|
. ' ' . socket_strerror(socket_last_error())); |
|
1806 |
|
@socket_close($s); |
|
1807 |
|
break; |
|
1808 |
|
} |
|
1809 |
|
|
|
1810 |
|
$k = 'chan-' . $job['id']; |
|
1811 |
|
rg_conn_new($k, $s); |
|
1812 |
|
$rg_conns[$k]['func_data'] = 'xchan'; |
|
1813 |
|
$rg_conns[$k]['jid'] = $job['id']; |
|
1814 |
|
rg_log($jid . ': Channel openned'); |
|
1815 |
|
} while (0); |
|
1816 |
|
} |
|
1817 |
|
|
880 |
1818 |
/* |
/* |
881 |
1819 |
* Artifacts parser |
* Artifacts parser |
882 |
1820 |
*/ |
*/ |
883 |
1821 |
function rg_job_artifacts_parse($root, $a) |
function rg_job_artifacts_parse($root, $a) |
884 |
1822 |
{ |
{ |
885 |
|
rg_log('DEBUG: artifacts text: ' . $a); |
|
|
1823 |
|
rg_log_debug('artifacts text: ' . $a); |
886 |
1824 |
|
|
887 |
1825 |
$ret = array('files' => array(), 'errors' => array(), 'total_size' => 0); |
$ret = array('files' => array(), 'errors' => array(), 'total_size' => 0); |
888 |
1826 |
while (1) { |
while (1) { |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
904 |
1842 |
if (strncmp($line, '#', 1) == 0) |
if (strncmp($line, '#', 1) == 0) |
905 |
1843 |
continue; |
continue; |
906 |
1844 |
|
|
907 |
|
rg_log('DEBUG: line=' . $line); |
|
|
1845 |
|
rg_log_debug('line=' . $line); |
908 |
1846 |
|
|
909 |
1847 |
if ((strncmp($line, ' ', 1) == 0) || (strncmp($line, "\t", 1) == 0)) { |
if ((strncmp($line, ' ', 1) == 0) || (strncmp($line, "\t", 1) == 0)) { |
910 |
|
rg_log('DEBUG: we have a sub path option'); |
|
|
1848 |
|
rg_log_debug('we have a sub path option'); |
911 |
1849 |
// we have an option for the previous declared file |
// we have an option for the previous declared file |
912 |
1850 |
if ($next_aid == 0) { |
if ($next_aid == 0) { |
913 |
1851 |
$ret['errors'][] = 'line ' . $line_no |
$ret['errors'][] = 'line ' . $line_no |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
933 |
1871 |
$make_array = TRUE; |
$make_array = TRUE; |
934 |
1872 |
if (strcmp($k, 'map_into_source') == 0) |
if (strcmp($k, 'map_into_source') == 0) |
935 |
1873 |
$make_array = TRUE; |
$make_array = TRUE; |
|
1874 |
|
if (strcmp($k, 'repo map') == 0) |
|
1875 |
|
$make_array = TRUE; |
936 |
1876 |
|
|
937 |
1877 |
if ($make_array) { |
if ($make_array) { |
938 |
1878 |
if (!isset($list[$aid]['para'][$para_i][$k])) |
if (!isset($list[$aid]['para'][$para_i][$k])) |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
944 |
1884 |
continue; |
continue; |
945 |
1885 |
} |
} |
946 |
1886 |
|
|
947 |
|
rg_log('DEBUG: We have a new path'); |
|
|
1887 |
|
rg_log_debug('We have a new path'); |
948 |
1888 |
$last = substr($line, -1); |
$last = substr($line, -1); |
949 |
1889 |
if (strcmp($last, '/') != 0) { |
if (strcmp($last, '/') != 0) { |
950 |
1890 |
// Search, maybe the file is sent again with different parameters |
// Search, maybe the file is sent again with different parameters |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
973 |
1913 |
$aid = $next_aid; |
$aid = $next_aid; |
974 |
1914 |
rg_log(' DEBUG: allocated aid ' . $aid); |
rg_log(' DEBUG: allocated aid ' . $aid); |
975 |
1915 |
} |
} |
976 |
|
//rg_log_ml('DEBUG: list: ' . rg_array2string($list)); |
|
|
1916 |
|
//rg_log_debug('list: ' . rg_array2string($list)); |
977 |
1917 |
|
|
978 |
1918 |
// Now, it is time to filter the list of files, if needed |
// Now, it is time to filter the list of files, if needed |
979 |
1919 |
$i = 0; |
$i = 0; |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
981 |
1921 |
unset($per_aid['para_i']); |
unset($per_aid['para_i']); |
982 |
1922 |
|
|
983 |
1923 |
if (empty($per_aid['para'])) { |
if (empty($per_aid['para'])) { |
984 |
|
rg_log('DEBUG: empty para, ignore entry [' . $per_aid['path'] . ']!'); |
|
|
1924 |
|
rg_log_debug('empty para, ignore entry [' . $per_aid['path'] . ']!'); |
985 |
1925 |
unset($list[$aid]); |
unset($list[$aid]); |
986 |
1926 |
continue; |
continue; |
987 |
1927 |
} |
} |
|
... |
... |
function rg_job_artifacts_parse($root, $a) |
1003 |
1943 |
continue; |
continue; |
1004 |
1944 |
} |
} |
1005 |
1945 |
|
|
1006 |
|
rg_log('DEBUG: we have a dir to match! path=' . $per_aid['path']); |
|
|
1946 |
|
rg_log_debug('we have a dir to match! path=' . $per_aid['path']); |
1007 |
1947 |
|
|
1008 |
1948 |
$d = rg_dir_load($root . '/' . $per_aid['path']); |
$d = rg_dir_load($root . '/' . $per_aid['path']); |
1009 |
1949 |
if ($d === FALSE) { |
if ($d === FALSE) { |
1010 |
|
rg_log('DEBUG: cannot load dir: ' . rg_util_error()); |
|
|
1950 |
|
rg_log(' DEBUG: cannot load dir: ' . rg_util_error()); |
1011 |
1951 |
continue; |
continue; |
1012 |
1952 |
} |
} |
1013 |
1953 |
|
|
1014 |
1954 |
foreach ($per_aid['para'] as $para_i => $per_para) { |
foreach ($per_aid['para'] as $para_i => $per_para) { |
1015 |
|
foreach ($per_para['map'] as $imap => $map) { |
|
1016 |
|
$last = substr($map, -1); |
|
1017 |
|
if (strcmp($last, '/') != 0) { |
|
1018 |
|
rg_log('DEBUG: map does not end in /. Bad! Ignore it!'); |
|
1019 |
|
unset($list[$aid]['para'][$para_i]['map'][$imap]); |
|
1020 |
|
continue; |
|
|
1955 |
|
if (isset($per_para['map'])) { |
|
1956 |
|
foreach ($per_para['map'] as $imap => $map) { |
|
1957 |
|
$last = substr($map, -1); |
|
1958 |
|
if (strcmp($last, '/') != 0) { |
|
1959 |
|
rg_log(' DEBUG: \'map\' does not end in \'/\'. Bad! Ignore it!'); |
|
1960 |
|
unset($list[$aid]['para'][$para_i]['map'][$imap]); |
|
1961 |
|
continue; |
|
1962 |
|
} |
1021 |
1963 |
} |
} |
1022 |
1964 |
} |
} |
1023 |
1965 |
|
|
1024 |
|
if (empty($list[$aid]['para'][$para_i]['map'])) { |
|
1025 |
|
rg_log('DEBUG: map is empty! Ignore whole para!'); |
|
1026 |
|
unset($list[$aid]['para'][$para_i]); |
|
1027 |
|
continue; |
|
1028 |
|
} |
|
1029 |
|
|
|
1030 |
|
if (!isset($per_para['regex'])) |
|
1031 |
|
$per_para['regex'] = '.*'; |
|
1032 |
|
|
|
1033 |
1966 |
foreach ($d as $f) { |
foreach ($d as $f) { |
1034 |
|
if (preg_match('/' . $per_para['regex'] . '/uD', $f) !== 1) { |
|
1035 |
|
rg_log('DEBUG: f=' . $f . ' does not match pattern ' . $per_para['regex']); |
|
|
1967 |
|
// TODO: What if $f is a dir? |
|
1968 |
|
if (isset($per_para['regex']) |
|
1969 |
|
&& (preg_match('/' . $per_para['regex'] . '/uD', $f) !== 1)) { |
|
1970 |
|
//rg_log(' DEBUG: f=' . $f . ' does not match pattern ' . $per_para['regex']); |
1036 |
1971 |
continue; |
continue; |
1037 |
1972 |
} |
} |
1038 |
1973 |
|
|
1039 |
1974 |
$r = @stat($root . '/' . $per_aid['path'] . $f); |
$r = @stat($root . '/' . $per_aid['path'] . $f); |
1040 |
1975 |
if ($r === FALSE) { |
if ($r === FALSE) { |
1041 |
|
rg_log('cannot stat file! Strange! Ignore it!'); |
|
|
1976 |
|
rg_log(' cannot stat file! Strange! Ignore it!'); |
1042 |
1977 |
continue; |
continue; |
1043 |
1978 |
} |
} |
1044 |
1979 |
|
|
|
... |
... |
function rg_job_upload_artifacts(&$job) |
1076 |
2011 |
|
|
1077 |
2012 |
rg_log('rg_job_upload_artifacts'); |
rg_log('rg_job_upload_artifacts'); |
1078 |
2013 |
|
|
1079 |
|
$f = $job['main'] . '/root/git/rocketgit/artifacts'; |
|
|
2014 |
|
$f = $job['main'] . '/root/build2/rocketgit/artifacts'; |
1080 |
2015 |
if (!file_exists($f)) { |
if (!file_exists($f)) { |
1081 |
|
rg_log('DEBUG: No artifacts file - no work to do.'); |
|
|
2016 |
|
rg_log('No artifacts file - no work to do [' . $f . ']'); |
1082 |
2017 |
// TODO: Should we update the builder with the new state? Yes! |
// TODO: Should we update the builder with the new state? Yes! |
1083 |
2018 |
$job['state'] = RG_JOB_FINISH; |
$job['state'] = RG_JOB_FINISH; |
1084 |
2019 |
$job['dirty'] = 1; |
$job['dirty'] = 1; |
|
... |
... |
function rg_job_upload_artifacts(&$job) |
1086 |
2021 |
} |
} |
1087 |
2022 |
|
|
1088 |
2023 |
$jid = $job['id']; |
$jid = $job['id']; |
1089 |
|
$root = $job['main'] . '/root/git/'; |
|
|
2024 |
|
$root = $job['main'] . '/root/target/'; |
1090 |
2025 |
$root_len = strlen($root); |
$root_len = strlen($root); |
1091 |
2026 |
|
|
1092 |
2027 |
if (!isset($job['artifacts'])) { |
if (!isset($job['artifacts'])) { |
|
... |
... |
function rg_job_upload_artifacts(&$job) |
1103 |
2038 |
if (isset($job['info_confirmed'])) |
if (isset($job['info_confirmed'])) |
1104 |
2039 |
break; |
break; |
1105 |
2040 |
|
|
1106 |
|
if (isset($job['last_reconnect']) && ($job['last_reconnect'] == $last_reconnect)) |
|
|
2041 |
|
if (isset($job['last_reconnect']) |
|
2042 |
|
&& ($job['last_reconnect'] == $last_reconnect)) |
1107 |
2043 |
break; |
break; |
1108 |
2044 |
|
|
1109 |
2045 |
$j = array( |
$j = array( |
|
... |
... |
function rg_job_extract_info(&$job) |
1256 |
2192 |
{ |
{ |
1257 |
2193 |
global $conf; |
global $conf; |
1258 |
2194 |
|
|
1259 |
|
rg_log_ml('DEBUG: extract_info: job: ' . rg_array2string($job)); |
|
|
2195 |
|
rg_log_debug('extract_info: job: ' . rg_array2string($job)); |
1260 |
2196 |
|
|
1261 |
2197 |
$jid = $job['id']; |
$jid = $job['id']; |
1262 |
2198 |
$emain = escapeshellarg($job['main']); |
$emain = escapeshellarg($job['main']); |
|
... |
... |
function rg_job_extract_info(&$job) |
1350 |
2286 |
else |
else |
1351 |
2287 |
$vm_start = intval($vm_start); |
$vm_start = intval($vm_start); |
1352 |
2288 |
|
|
|
2289 |
|
$pkg_subrepo_dirty = array(); |
|
2290 |
|
$r = @file_get_contents($job['main'] . '/root/build2/pkg_subrepo_dirty'); |
|
2291 |
|
if ($r !== FALSE) { |
|
2292 |
|
$r = trim($r); |
|
2293 |
|
$x = explode("\n", $r); |
|
2294 |
|
// remove duplicates |
|
2295 |
|
$list = array(); |
|
2296 |
|
foreach ($x as $id) |
|
2297 |
|
$list[$id] = 1; |
|
2298 |
|
foreach ($list as $id => $junk) |
|
2299 |
|
$pkg_subrepo_dirty[] = $id; |
|
2300 |
|
} |
|
2301 |
|
|
1353 |
2302 |
$job['status'] = array( |
$job['status'] = array( |
1354 |
2303 |
'vm_start' => $vm_start, |
'vm_start' => $vm_start, |
1355 |
2304 |
'build_sh_start' => $build_sh_start, |
'build_sh_start' => $build_sh_start, |
|
... |
... |
function rg_job_extract_info(&$job) |
1358 |
2307 |
'start' => @trim(file_get_contents($job['main'] . '/root/T_START')), |
'start' => @trim(file_get_contents($job['main'] . '/root/T_START')), |
1359 |
2308 |
'net_ok' => @trim(file_get_contents($job['main'] . '/root/T_NET_OK')), |
'net_ok' => @trim(file_get_contents($job['main'] . '/root/T_NET_OK')), |
1360 |
2309 |
'pkgs_ok' => @trim(file_get_contents($job['main'] . '/root/T_PKGS_OK')), |
'pkgs_ok' => @trim(file_get_contents($job['main'] . '/root/T_PKGS_OK')), |
1361 |
|
'done' => @trim(file_get_contents($job['main'] . '/root/T_DONE')), |
|
1362 |
|
'labels' => $labels |
|
|
2310 |
|
'done' => @trim(file_get_contents($job['main'] . '/root/T_DONE')) || time(), |
|
2311 |
|
'labels' => $labels, |
|
2312 |
|
'pkg_subrepo_dirty' => $pkg_subrepo_dirty |
1363 |
2313 |
); |
); |
1364 |
2314 |
|
|
1365 |
2315 |
$job['status']['cmds'] = array(); |
$job['status']['cmds'] = array(); |
|
2316 |
|
if (!isset($job['cmds'])) |
|
2317 |
|
$job['cmds'] = array(); |
1366 |
2318 |
foreach ($job['cmds'] as $cmd => $i) { |
foreach ($job['cmds'] as $cmd => $i) { |
1367 |
2319 |
if (empty($i['cmd'])) |
if (empty($i['cmd'])) |
1368 |
2320 |
continue; |
continue; |
|
... |
... |
function rg_job_extract_info(&$job) |
1381 |
2333 |
unset($job['url']); |
unset($job['url']); |
1382 |
2334 |
unset($job['head']); |
unset($job['head']); |
1383 |
2335 |
unset($job['env']); |
unset($job['env']); |
1384 |
|
rg_log('DEBUG: state set to JOB_ARTIFACTS'); |
|
|
2336 |
|
rg_log_debug('state set to JOB_ARTIFACTS'); |
1385 |
2337 |
$job['state'] = RG_JOB_ARTIFACTS; |
$job['state'] = RG_JOB_ARTIFACTS; |
1386 |
2338 |
break; |
break; |
1387 |
2339 |
} |
} |
1388 |
2340 |
|
|
1389 |
2341 |
if (!empty($job['error'])) { |
if (!empty($job['error'])) { |
1390 |
|
rg_log('DEBUG: state set to JOB_ERROR (' . $job['error'] . ')'); |
|
|
2342 |
|
rg_log_debug('state set to JOB_ERROR (' . $job['error'] . ')'); |
1391 |
2343 |
$job['state'] = RG_JOB_ERROR; |
$job['state'] = RG_JOB_ERROR; |
1392 |
2344 |
return; |
return; |
1393 |
2345 |
} |
} |
|
... |
... |
function rg_job_extract_info(&$job) |
1399 |
2351 |
function vm_extract_info($name) |
function vm_extract_info($name) |
1400 |
2352 |
{ |
{ |
1401 |
2353 |
$ret = FALSE; |
$ret = FALSE; |
1402 |
|
while (1) { |
|
|
2354 |
|
do { |
1403 |
2355 |
$cmd = 'virsh domstats --raw ' . escapeshellarg($name); |
$cmd = 'virsh domstats --raw ' . escapeshellarg($name); |
1404 |
2356 |
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
1405 |
2357 |
if ($r['ok'] != 1) { |
if ($r['ok'] != 1) { |
|
... |
... |
function vm_extract_info($name) |
1407 |
2359 |
break; |
break; |
1408 |
2360 |
} |
} |
1409 |
2361 |
|
|
1410 |
|
//rg_log_ml('DEBUG: domstats: ' . rg_array2string($r['data'])); |
|
|
2362 |
|
rg_debug() && rg_log_debug('domstats: ' . rg_array2string($r['data'])); |
1411 |
2363 |
$data = array(); |
$data = array(); |
1412 |
2364 |
$t = explode("\n", $r['data']); |
$t = explode("\n", $r['data']); |
1413 |
2365 |
foreach ($t as $line) { |
foreach ($t as $line) { |
|
... |
... |
function vm_extract_info($name) |
1419 |
2371 |
} |
} |
1420 |
2372 |
|
|
1421 |
2373 |
$ret = array(); |
$ret = array(); |
|
2374 |
|
if (!isset($data['net.0.rx.bytes'])) |
|
2375 |
|
break; |
|
2376 |
|
|
1422 |
2377 |
$ret['rx_bytes'] = $data['net.0.rx.bytes']; |
$ret['rx_bytes'] = $data['net.0.rx.bytes']; |
1423 |
2378 |
$ret['rx_pkts'] = $data['net.0.rx.pkts']; |
$ret['rx_pkts'] = $data['net.0.rx.pkts']; |
1424 |
2379 |
$ret['tx_bytes'] = $data['net.0.tx.bytes']; |
$ret['tx_bytes'] = $data['net.0.tx.bytes']; |
|
... |
... |
function vm_extract_info($name) |
1434 |
2389 |
$ret['cpu_time_ns'] = $data['cpu.time']; |
$ret['cpu_time_ns'] = $data['cpu.time']; |
1435 |
2390 |
$ret['ballon_current_mib'] = intval($data['balloon.current'] / 1024); |
$ret['ballon_current_mib'] = intval($data['balloon.current'] / 1024); |
1436 |
2391 |
$ret['ballon_rss_mib'] = intval($data['balloon.rss'] / 1024); |
$ret['ballon_rss_mib'] = intval($data['balloon.rss'] / 1024); |
1437 |
|
break; |
|
1438 |
|
} |
|
|
2392 |
|
} while(0); |
1439 |
2393 |
|
|
1440 |
2394 |
return $ret; |
return $ret; |
1441 |
2395 |
} |
} |
|
... |
... |
function rg_worker_connect() |
1556 |
2510 |
} |
} |
1557 |
2511 |
|
|
1558 |
2512 |
|
|
|
2513 |
|
rg_prof_start('MAIN'); |
|
2514 |
|
|
|
2515 |
|
if (!isset($_SERVER['argv'][1])) { |
|
2516 |
|
$id = 'main'; |
|
2517 |
|
} else { |
|
2518 |
|
$id = $_SERVER['argv'][1]; |
|
2519 |
|
} |
|
2520 |
|
|
|
2521 |
|
if (!isset($_SERVER['argv'][2])) { |
|
2522 |
|
$conf_file = '/etc/rocketgit/worker.conf'; |
|
2523 |
|
} else { |
|
2524 |
|
$conf_file = $_SERVER['argv'][2]; |
|
2525 |
|
} |
|
2526 |
|
|
|
2527 |
|
rg_log_set_file($rg_log_dir . '/worker-' . $id . '.log'); |
|
2528 |
|
rg_log_set_sid("000000"); // to spread the logs |
|
2529 |
|
|
|
2530 |
|
$rg_cache_core_enable = FALSE; |
|
2531 |
|
|
1559 |
2532 |
umask(0007); |
umask(0007); |
|
2533 |
|
|
1560 |
2534 |
load_config($conf_file); |
load_config($conf_file); |
1561 |
2535 |
rg_log('id is [' . $id . ']'); |
rg_log('id is [' . $id . ']'); |
1562 |
2536 |
$conf['id'] = $id; |
$conf['id'] = $id; |
|
... |
... |
if (!empty($jobs)) |
1573 |
2547 |
|
|
1574 |
2548 |
rg_worker_connect(); |
rg_worker_connect(); |
1575 |
2549 |
|
|
1576 |
|
$pid_to_jid = array(); |
|
|
2550 |
|
$pid_to_jid = array(); // TODO: what happens to this after restart? |
1577 |
2551 |
while(1) { |
while(1) { |
1578 |
2552 |
rg_conn_wait(1); |
rg_conn_wait(1); |
1579 |
2553 |
|
|
|
... |
... |
while(1) { |
1593 |
2567 |
} |
} |
1594 |
2568 |
|
|
1595 |
2569 |
$jid = $pid_to_jid[$pid]; |
$jid = $pid_to_jid[$pid]; |
1596 |
|
//rg_log('Pid ' . $pid . ' exited (job ' . $jid . ')' |
|
1597 |
|
// . ' with status ' . $status . '!'); |
|
1598 |
2570 |
unset($pid_to_jid[$pid]); |
unset($pid_to_jid[$pid]); |
1599 |
|
rg_log($jid . ': Switch to state STARTED'); |
|
1600 |
|
$jobs[$jid]['state'] = RG_JOB_STARTED; |
|
1601 |
|
$jobs[$jid]['dirty'] = 1; |
|
1602 |
|
$stats['jobs']++; |
|
|
2571 |
|
|
|
2572 |
|
$jobs[$jid]['helper_stop'] = time(); |
|
2573 |
|
|
|
2574 |
|
if ($status != 0) { |
|
2575 |
|
rg_log($jid . ': DEBUG: Pid ' . $pid . ' exited' |
|
2576 |
|
. ' with status ' . $status . '! Switch to DONE'); |
|
2577 |
|
$jobs[$jid]['error'] = 'helper error'; |
|
2578 |
|
$jobs[$jid]['error2'] = 'helper error'; |
|
2579 |
|
$jobs[$jid]['state'] = RG_JOB_DONE; |
|
2580 |
|
$jobs[$jid]['dirty'] = 1; |
|
2581 |
|
save_job($job); |
|
2582 |
|
} |
1603 |
2583 |
} |
} |
1604 |
2584 |
|
|
1605 |
2585 |
// Verify if VMs finished |
// Verify if VMs finished |
1606 |
2586 |
//if (!empty($jobs)) |
//if (!empty($jobs)) |
1607 |
|
// rg_log_ml('DEBUG: jobs: ' . print_r($jobs, TRUE)); |
|
|
2587 |
|
// rg_log_debug('jobs: ' . print_r($jobs, TRUE)); |
1608 |
2588 |
$vms_loaded = FALSE; |
$vms_loaded = FALSE; |
1609 |
2589 |
foreach ($jobs as $jid => &$job) { |
foreach ($jobs as $jid => &$job) { |
1610 |
2590 |
//rg_log($jid . ': DEBUG: state \'' . job_state($job['state']) . '\''); |
//rg_log($jid . ': DEBUG: state \'' . job_state($job['state']) . '\''); |
|
... |
... |
while(1) { |
1623 |
2603 |
rg_log($jid . ': Switch to state STARTED (from INIT)'); |
rg_log($jid . ': Switch to state STARTED (from INIT)'); |
1624 |
2604 |
$job['state'] = RG_JOB_STARTED; |
$job['state'] = RG_JOB_STARTED; |
1625 |
2605 |
$job['dirty'] = 1; |
$job['dirty'] = 1; |
|
2606 |
|
xchan_open($job); |
1626 |
2607 |
save_job($job); |
save_job($job); |
1627 |
2608 |
break; |
break; |
1628 |
2609 |
} |
} |
|
... |
... |
while(1) { |
1639 |
2620 |
$job['state'] = RG_JOB_HELPER_STARTED; |
$job['state'] = RG_JOB_HELPER_STARTED; |
1640 |
2621 |
$job['start'] = time(); |
$job['start'] = time(); |
1641 |
2622 |
$job['dirty'] = 1; |
$job['dirty'] = 1; |
|
2623 |
|
$job['helper_pid'] = $pid; |
|
2624 |
|
$job['helper_start'] = time(); |
1642 |
2625 |
save_job($job); |
save_job($job); |
1643 |
2626 |
$pid_to_jid[$pid] = $jid; |
$pid_to_jid[$pid] = $jid; |
1644 |
2627 |
|
|
|
... |
... |
while(1) { |
1654 |
2637 |
// We need to wait for the VM to start |
// We need to wait for the VM to start |
1655 |
2638 |
$k = array_search($name, $vms); |
$k = array_search($name, $vms); |
1656 |
2639 |
if ($k !== FALSE) { |
if ($k !== FALSE) { |
1657 |
|
rg_log($jid . ': VM found, switch to state STARTED'); |
|
|
2640 |
|
rg_log($jid . ': VM is started, switch to state STARTED'); |
1658 |
2641 |
$job['state'] = RG_JOB_STARTED; |
$job['state'] = RG_JOB_STARTED; |
|
2642 |
|
xchan_open($job); |
|
2643 |
|
$stats['jobs']++; |
|
2644 |
|
} else if (!isset($job['helper_start']) |
|
2645 |
|
|| ($job['helper_start'] + 30 < time())) { |
|
2646 |
|
// TODO: we should check if pid is still valid? |
|
2647 |
|
rg_log($jid . ': Waited 30s without the VM to show up!' |
|
2648 |
|
. ' Set state to INIT.'); |
|
2649 |
|
$job['state'] = RG_JOB_INIT; |
|
2650 |
|
continue; |
1659 |
2651 |
} |
} |
1660 |
|
// TODO: how much to wait in this state? |
|
1661 |
2652 |
} |
} |
1662 |
2653 |
|
|
1663 |
2654 |
if ($job['state'] == RG_JOB_STARTED) { |
if ($job['state'] == RG_JOB_STARTED) { |
|
... |
... |
while(1) { |
1674 |
2665 |
$job['stats'] = $r; |
$job['stats'] = $r; |
1675 |
2666 |
|
|
1676 |
2667 |
// TODO: timeout must be controlled by user |
// TODO: timeout must be controlled by user |
1677 |
|
if ($job['start'] + 6 * 3600 < time()) { |
|
1678 |
|
$cmd = 'virsh destroy rocketgit-j-' . $jid; |
|
|
2668 |
|
if ($job['start'] + 1 * 3600 < time()) { |
|
2669 |
|
rg_log($jid . ': More than 6 hours without progress! Kill VM!'); |
|
2670 |
|
$_name = 'rg-worker-' . $conf['id'] . '-' . $jid; |
|
2671 |
|
$cmd = 'virsh destroy ' . escapeshellarg($_name); |
1679 |
2672 |
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
$r = rg_exec($cmd, '', FALSE, FALSE, FALSE); |
1680 |
2673 |
if ($r['ok'] != 1) { |
if ($r['ok'] != 1) { |
1681 |
2674 |
// If error, probably the machine was not running, so, this is just a warning |
// If error, probably the machine was not running, so, this is just a warning |
1682 |
2675 |
// TODO: But we set an error! |
// TODO: But we set an error! |
1683 |
2676 |
$job['error'] = 'Could not destroy: ' . $r['errmsg']; |
$job['error'] = 'Could not destroy: ' . $r['errmsg']; |
1684 |
|
rg_log('Error: ' . $job['error']); |
|
|
2677 |
|
rg_log($jid . ': Error: ' . $job['error']); |
1685 |
2678 |
} |
} |
1686 |
2679 |
} |
} |
1687 |
2680 |
continue; |
continue; |
|
... |
... |
while(1) { |
1712 |
2705 |
rg_internal_error('Cannot unmount [' . $job['main'] |
rg_internal_error('Cannot unmount [' . $job['main'] |
1713 |
2706 |
. '/root]: ' . $r['errmsg'] . '!'); |
. '/root]: ' . $r['errmsg'] . '!'); |
1714 |
2707 |
|
|
|
2708 |
|
@socket_close($job['chan']); |
|
2709 |
|
|
1715 |
2710 |
$r = rg_del_tree($job['main']); |
$r = rg_del_tree($job['main']); |
1716 |
2711 |
if ($r === FALSE) |
if ($r === FALSE) |
1717 |
2712 |
rg_log($jid . ': cannot delete tree: ' . rg_util_error()); |
rg_log($jid . ': cannot delete tree: ' . rg_util_error()); |