jobserver.py

  1#!/usr/bin/env python3
  2# SPDX-License-Identifier: GPL-2.0+
  3#
  4# pylint: disable=C0103,C0209
  5#
  6#
  7
  8"""
  9Interacts with the POSIX jobserver during the Kernel build time.
 10
 11A "normal" jobserver task, like the one initiated by a make subrocess would do:
 12
 13    - open read/write file descriptors to communicate with the job server;
 14    - ask for one slot by calling:
 15        claim = os.read(reader, 1)
 16    - when the job finshes, call:
 17        os.write(writer, b"+")  # os.write(writer, claim)
 18
 19Here, the goal is different: This script aims to get the remaining number
 20of slots available, using all of them to run a command which handle tasks in
 21parallel. To to that, it has a loop that ends only after there are no
 22slots left. It then increments the number by one, in order to allow a
 23call equivalent to make -j$((claim+1)), e.g. having a parent make creating
 24$claim child to do the actual work.
 25
 26The end goal here is to keep the total number of build tasks under the
 27limit established by the initial make -j$n_proc call.
 28
 29See:
 30    https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
 31"""
 32
 33import errno
 34import os
 35import subprocess
 36import sys
 37
 38class JobserverExec:
 39    """
 40    Claim all slots from make using POSIX Jobserver.
 41
 42    The main methods here are:
 43    - open(): reserves all slots;
 44    - close(): method returns all used slots back to make;
 45    - run(): executes a command setting PARALLELISM=<available slots jobs + 1>
 46    """
 47
 48    def __init__(self):
 49        """Initialize internal vars"""
 50        self.claim = 0
 51        self.jobs = b""
 52        self.reader = None
 53        self.writer = None
 54        self.is_open = False
 55
 56    def open(self):
 57        """Reserve all available slots to be claimed later on"""
 58
 59        if self.is_open:
 60            return
 61
 62        try:
 63            # Fetch the make environment options.
 64            flags = os.environ["MAKEFLAGS"]
 65            # Look for "--jobserver=R,W"
 66            # Note that GNU Make has used --jobserver-fds and --jobserver-auth
 67            # so this handles all of them.
 68            opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
 69
 70            # Parse out R,W file descriptor numbers and set them nonblocking.
 71            # If the MAKEFLAGS variable contains multiple instances of the
 72            # --jobserver-auth= option, the last one is relevant.
 73            fds = opts[-1].split("=", 1)[1]
 74
 75            # Starting with GNU Make 4.4, named pipes are used for reader
 76            # and writer.
 77            # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
 78            _, _, path = fds.partition("fifo:")
 79
 80            if path:
 81                self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
 82                self.writer = os.open(path, os.O_WRONLY)
 83            else:
 84                self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
 85                # Open a private copy of reader to avoid setting nonblocking
 86                # on an unexpecting process with the same reader fd.
 87                self.reader = os.open("/proc/self/fd/%d" % (self.reader),
 88                                      os.O_RDONLY | os.O_NONBLOCK)
 89
 90            # Read out as many jobserver slots as possible
 91            while True:
 92                try:
 93                    slot = os.read(self.reader, 8)
 94                    self.jobs += slot
 95                except (OSError, IOError) as e:
 96                    if e.errno == errno.EWOULDBLOCK:
 97                        # Stop at the end of the jobserver queue.
 98                        break
 99                    # If something went wrong, give back the jobs.
100                    if self.jobs:
101                        os.write(self.writer, self.jobs)
102                    raise e
103
104            # Add a bump for our caller's reserveration, since we're just going
105            # to sit here blocked on our child.
106            self.claim = len(self.jobs) + 1
107
108        except (KeyError, IndexError, ValueError, OSError, IOError):
109            # Any missing environment strings or bad fds should result in just
110            # not being parallel.
111            self.claim = None
112
113        self.is_open = True
114
115    def close(self):
116        """Return all reserved slots to Jobserver"""
117
118        if not self.is_open:
119            return
120
121        # Return all the reserved slots.
122        if len(self.jobs):
123            os.write(self.writer, self.jobs)
124
125        self.is_open = False
126
127    def __enter__(self):
128        self.open()
129        return self
130
131    def __exit__(self, exc_type, exc_value, exc_traceback):
132        self.close()
133
134    def run(self, cmd, *args, **pwargs):
135        """
136        Run a command setting PARALLELISM env variable to the number of
137        available job slots (claim) + 1, e.g. it will reserve claim slots
138        to do the actual build work, plus one to monitor its children.
139        """
140        self.open()             # Ensure that self.claim is set
141
142        # We can only claim parallelism if there was a jobserver (i.e. a
143        # top-level "-jN" argument) and there were no other failures. Otherwise
144        # leave out the environment variable and let the child figure out what
145        # is best.
146        if self.claim:
147            os.environ["PARALLELISM"] = str(self.claim)
148
149        return subprocess.call(cmd, *args, **pwargs)