Support for GNU Make jobserver (alternative implementation)

Makefile

@@ -4,7 +4,7 @@
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/share/man
-ALL_CFLAGS=$(CFLAGS) -std=c99 -Wall -Wextra -Wshadow -Wmissing-prototypes -Wpedantic -Wno-unused-parameter
+ALL_CFLAGS=$(CFLAGS) -std=c99 -Wall -Wextra -Wshadow -Wmissing-prototypes -Wpedantic -Wno-unused-parameter -pthread
 OBJ=\
 	build.o\
 	deps.o\
@@ -15,6 +15,7 @@ OBJ=\
 	parse.o\
 	samu.o\
 	scan.o\
+	token.o\
 	tool.o\
 	tree.o\
 	util.o
@@ -28,6 +29,7 @@ HDR=\
 	log.h\
 	parse.h\
 	scan.h\
+	token.h\
 	tool.h\
 	tree.h\
 	util.h

build.c

@@ -2,12 +2,14 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
+#include <limits.h>
 #include <poll.h>
 #include <signal.h>
 #include <spawn.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
@@ -16,6 +18,7 @@
 #include "env.h"
 #include "graph.h"
 #include "log.h"
+#include "token.h"
 #include "util.h"
 
 struct job {
@@ -34,6 +37,10 @@ static size_t nstarted, nfinished, ntotal;
 static bool consoleused;
 static struct timespec starttime;
 
+/* for signal handling */
+static volatile bool killall;
+static volatile int stopsig;
+
 void
 buildreset(void)
 {
@@ -115,6 +122,11 @@ queue(struct edge *e)
 	*front = e;
 }
 
+static bool have_work(void)
+{
+	return work && !stopsig;
+}
+
 void
 buildadd(struct node *n)
 {
@@ -453,7 +465,8 @@ jobdone(struct job *j)
 			j->failed = true;
 		}
 	} else if (WIFSIGNALED(status)) {
-		warn("job terminated due to signal %d: %s", WTERMSIG(status), j->cmd->s);
+		if (stopsig != WTERMSIG(status))
+			warn("job terminated due to signal %d: %s", WTERMSIG(status), j->cmd->s);
 		j->failed = true;
 	} else {
 		/* cannot happen according to POSIX */
@@ -465,6 +478,8 @@ jobdone(struct job *j)
 		fwrite(j->buf.data, 1, j->buf.len, stdout);
 	j->buf.len = 0;
 	e = j->edge;
+	j->edge = NULL;
+
 	if (e->pool) {
 		p = e->pool;
 
@@ -492,6 +507,9 @@ jobwork(struct job *j)
 	size_t newcap;
 	ssize_t n;
 
+	if (killall)
+		goto kill;
+
 	if (j->buf.cap - j->buf.len < BUFSIZ / 2) {
 		newcap = j->buf.cap + BUFSIZ;
 		newdata = realloc(j->buf.data, newcap);
@@ -538,47 +556,86 @@ queryload(void)
 #endif
 }
 
+/*
+ * SIGINT handler: do not start any more processes; SIGINT is sent
+ * to all children because samurai is a process group leader, so
+ * they will stop on their own.
+ */
+static void inthandler(int sig)
+{
+	stopsig = SIGINT;
+}
+
+/*
+ * SIGTERM handler: do not start any more processes, and kill
+ * currently running ones.
+ */
+static void termhandler(int sig)
+{
+	killall = true;
+	stopsig = SIGTERM;
+}
+
 void
 build(void)
 {
 	struct job *jobs = NULL;
 	struct pollfd *fds = NULL;
 	size_t i, next = 0, jobslen = 0, maxjobs = buildopts.maxjobs, numjobs = 0, numfail = 0;
 	struct edge *e;
+	int jobserver_rfd;
+	struct sigaction sa;
 
 	if (ntotal == 0) {
 		warn("nothing to do");
 		return;
 	}
 
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = termhandler;
+	sa.sa_flags = SA_RESTART;
+	sigaction(SIGTERM, &sa, NULL);
+
+	sa.sa_handler = inthandler;
+	sa.sa_flags = SA_RESTART;
+	sigaction(SIGINT, &sa, NULL);
+
 	clock_gettime(CLOCK_MONOTONIC, &starttime);
 	formatstatus(NULL, 0);
+	jobserver_rfd = tokeninit();
+
+	/* if running under job server start as many jobs as allowed */
+	if (jobserver_rfd != -1)
+		buildopts.maxjobs = INT_MAX;
 
 	nstarted = 0;
 	for (;;) {
 		/* limit number of of jobs based on load */
 		if (buildopts.maxload)
 			maxjobs = queryload() > buildopts.maxload ? 1 : buildopts.maxjobs;
 		/* start ready edges */
-		while (work && numjobs < maxjobs && numfail < buildopts.maxfail) {
+		while (have_work() && numjobs < maxjobs && numfail < buildopts.maxfail) {
 			e = work;
-			work = work->worknext;
 			if (e->rule != &phonyrule && buildopts.dryrun) {
 				++nstarted;
 				printstatus(e, edgevar(e, "command", true));
 				++nfinished;
 			}
 			if (e->rule == &phonyrule || buildopts.dryrun) {
+				work = work->worknext;
 				for (i = 0; i < e->nout; ++i)
 					nodedone(e->out[i], false);
 				continue;
 			}
+			if (!tokenget(e))
+				break;
+			work = work->worknext;
 			if (next == jobslen) {
 				jobslen = jobslen ? jobslen * 2 : 8;
 				if (jobslen > buildopts.maxjobs)
 					jobslen = buildopts.maxjobs;
 				jobs = xreallocarray(jobs, jobslen, sizeof(jobs[0]));
-				fds = xreallocarray(fds, jobslen, sizeof(fds[0]));
+				fds = xreallocarray(fds, jobslen + 1, sizeof(fds[0]));
 				for (i = next; i < jobslen; ++i) {
 					jobs[i].buf.data = NULL;
 					jobs[i].buf.len = 0;
@@ -587,6 +644,8 @@ build(void)
 					fds[i].fd = -1;
 					fds[i].events = POLLIN;
 				}
+				fds[jobslen].fd = jobserver_rfd;
+				fds[jobslen].events = POLLIN;
 			}
 			fds[next].fd = jobstart(&jobs[next], e);
 			if (fds[next].fd < 0) {
@@ -597,19 +656,32 @@ build(void)
 				++numjobs;
 			}
 		}
-		if (numjobs == 0)
+		if (!have_work() && !numjobs)
 			break;
-		if (poll(fds, jobslen, 5000) < 0)
+		/*
+		 * the last slot is for the jobserver and is only used to
+		 * kick samurai out of poll()
+		 */
+		if (poll(fds, jobslen + 1, 5000) < 0 && errno != EINTR)
 			fatal("poll:");
 		for (i = 0; i < jobslen; ++i) {
-			if (!fds[i].revents || jobwork(&jobs[i]))
+			/* active job? */
+			if (!jobs[i].edge)
 				continue;
-			--numjobs;
-			jobs[i].next = next;
-			fds[i].fd = -1;
-			next = i;
-			if (jobs[i].failed)
-				++numfail;
+
+			/* anything to do? */
+			if (!killall && !fds[i].revents)
+				continue;
+
+			if (!jobwork(&jobs[i])) {
+				tokenput();
+				--numjobs;
+				jobs[i].next = next;
+				fds[i].fd = -1;
+				next = i;
+				if (jobs[i].failed)
+					++numfail;
+			}
 		}
 	}
 	for (i = 0; i < jobslen; ++i)
@@ -619,6 +691,10 @@ build(void)
 	if (numfail > 0) {
 		if (numfail < buildopts.maxfail)
 			fatal("cannot make progress due to previous errors");
+		else if (stopsig == SIGINT)
+			fatal("interrupted by user");
+		else if (stopsig)
+			fatal("interrupted by signal %d", stopsig);
 		else if (numfail > 1)
 			fatal("subcommands failed");
 		else

graph.h

@@ -66,6 +66,9 @@ struct edge {
 	struct edge *worknext;
 	/* used for alledges linked list */
 	struct edge *allnext;
+
+	/* has the edge requested a token from the job server */
+	_Bool reserve;
 };
 
 void graphinit(void);