def do_packagedata_setscene(d): sstate_setscene(d) do_packagedata_setscene(d) def sstate_setscene(d): shared_state = sstate_state_fromvars(d) accelerate = sstate_installpkg(shared_state, d) if not accelerate: msg = "No sstate archive obtainable, will run full task instead." bb.warn(msg) raise bb.BBHandledException(msg) def sstate_state_fromvars(d, task = None): if task is None: task = d.getVar('BB_CURRENTTASK') if not task: bb.fatal("sstate code running without task context?!") task = task.replace("_setscene", "") if task.startswith("do_"): task = task[3:] inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split() outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split() plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split() lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split() lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split() interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split() fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or "" if not task or len(inputs) != len(outputs): bb.fatal("sstate variables not setup correctly?!") if task == "populate_lic": d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") d.setVar("SSTATE_EXTRAPATH", "") d.setVar('SSTATE_EXTRAPATHWILDCARD', "") ss = sstate_init(task, d) for i in range(len(inputs)): sstate_add(ss, inputs[i], outputs[i], d) ss['lockfiles'] = lockfiles ss['lockfiles-shared'] = lockfilesshared ss['plaindirs'] = plaindirs ss['interceptfuncs'] = interceptfuncs ss['fixmedir'] = fixmedir return ss def sstate_installpkg(ss, d): from oe.gpg_sign import get_signer sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) d.setVar("SSTATE_CURRTASK", ss['task']) sstatefetch = d.getVar('SSTATE_PKGNAME') sstatepkg = d.getVar('SSTATE_PKG') verify_sig = bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False) if not os.path.exists(sstatepkg) or (verify_sig and not os.path.exists(sstatepkg + '.sig')): pstaging_fetch(sstatefetch, d) if not os.path.isfile(sstatepkg): bb.note("Sstate package %s does not exist" % sstatepkg) return False sstate_clean(ss, d) d.setVar('SSTATE_INSTDIR', sstateinst) if verify_sig: if not os.path.isfile(sstatepkg + '.sig'): bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg) return False signer = get_signer(d, 'local') if not signer.verify(sstatepkg + '.sig', d.getVar("SSTATE_VALID_SIGS")): bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg) return False # Empty sstateinst directory, ensure its clean if os.path.exists(sstateinst): oe.path.remove(sstateinst) bb.utils.mkdirhier(sstateinst) sstateinst = d.getVar("SSTATE_INSTDIR") d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']: # All hooks should run in the SSTATE_INSTDIR bb.build.exec_func(f, d, (sstateinst,)) return sstate_installpkgdir(ss, d) def pstaging_fetch(sstatefetch, d): import bb.fetch2 # Only try and fetch if the user has configured a mirror mirrors = d.getVar('SSTATE_MIRRORS') if not mirrors: return # Copy the data object and override DL_DIR and SRC_URI localdata = bb.data.createCopy(d) dldir = localdata.expand("${SSTATE_DIR}") bb.utils.mkdirhier(dldir) localdata.delVar('MIRRORS') localdata.setVar('FILESPATH', dldir) localdata.setVar('DL_DIR', dldir) localdata.setVar('PREMIRRORS', mirrors) # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, # we'll want to allow network access for the current set of fetches. if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \ bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')): localdata.delVar('BB_NO_NETWORK') # Try a fetch from the sstate mirror, if it fails just return and # we will build the package uris = ['file://{0};downloadfilename={0}'.format(sstatefetch), 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)] if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False): uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)] for srcuri in uris: localdata.delVar('SRC_URI') localdata.setVar('SRC_URI', srcuri) try: fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) fetcher.checkstatus() fetcher.download() except bb.fetch2.BBFetchException: pass def sstate_installpkgdir(ss, d): import oe.path import subprocess sstateinst = d.getVar("SSTATE_INSTDIR") d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split(): # All hooks should run in the SSTATE_INSTDIR bb.build.exec_func(f, d, (sstateinst,)) def prepdir(dir): # remove dir if it exists, ensure any parent directories do exist if os.path.exists(dir): oe.path.remove(dir) bb.utils.mkdirhier(dir) oe.path.remove(dir) for state in ss['dirs']: prepdir(state[1]) bb.utils.rename(sstateinst + state[0], state[1]) sstate_install(ss, d) for plain in ss['plaindirs']: workdir = d.getVar('WORKDIR') sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared") src = sstateinst + "/" + plain.replace(workdir, '') if sharedworkdir in plain: src = sstateinst + "/" + plain.replace(sharedworkdir, '') dest = plain bb.utils.mkdirhier(src) prepdir(dest) bb.utils.rename(src, dest) return True def sstate_init(task, d): ss = {} ss['task'] = task ss['dirs'] = [] ss['plaindirs'] = [] ss['lockfiles'] = [] ss['lockfiles-shared'] = [] return ss def sstate_clean(ss, d): import oe.path import glob d2 = d.createCopy() stamp_clean = d.getVar("STAMPCLEAN") extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info') if extrainf: d2.setVar("SSTATE_MANMACH", extrainf) wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf) else: wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task']) manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task']) if os.path.exists(manifest): locks = [] for lock in ss['lockfiles-shared']: locks.append(bb.utils.lockfile(lock)) for lock in ss['lockfiles']: locks.append(bb.utils.lockfile(lock)) sstate_clean_manifest(manifest, d, canrace=True) for lock in locks: bb.utils.unlockfile(lock) # Remove the current and previous stamps, but keep the sigdata. # # The glob() matches do_task* which may match multiple tasks, for # example: do_package and do_package_write_ipk, so we need to # exactly match *.do_task.* and *.do_task_setscene.* rm_stamp = '.do_%s.' % ss['task'] rm_setscene = '.do_%s_setscene.' % ss['task'] # For BB_SIGNATURE_HANDLER = "noop" rm_nohash = ".do_%s" % ss['task'] for stfile in glob.glob(wildcard_stfile): # Keep the sigdata if ".sigdata." in stfile or ".sigbasedata." in stfile: continue # Preserve taint files in the stamps directory if stfile.endswith('.taint'): continue if rm_stamp in stfile or rm_setscene in stfile or \ stfile.endswith(rm_nohash): oe.path.remove(stfile) def sstate_add(ss, source, dest, d): if not source.endswith("/"): source = source + "/" if not dest.endswith("/"): dest = dest + "/" source = os.path.normpath(source) dest = os.path.normpath(dest) srcbase = os.path.basename(source) ss['dirs'].append([srcbase, source, dest]) return ss def sstate_install(ss, d): import oe.path import oe.sstatesig import subprocess sharedfiles = [] shareddirs = [] bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d) if os.access(manifest, os.R_OK): bb.fatal("Package already staged (%s)?!" % manifest) d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm") locks = [] for lock in ss['lockfiles-shared']: locks.append(bb.utils.lockfile(lock, True)) for lock in ss['lockfiles']: locks.append(bb.utils.lockfile(lock)) for state in ss['dirs']: bb.debug(2, "Staging files from %s to %s" % (state[1], state[2])) for walkroot, dirs, files in os.walk(state[1]): for file in files: srcpath = os.path.join(walkroot, file) dstpath = srcpath.replace(state[1], state[2]) #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath)) sharedfiles.append(dstpath) for dir in dirs: srcdir = os.path.join(walkroot, dir) dstdir = srcdir.replace(state[1], state[2]) #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir)) if os.path.islink(srcdir): sharedfiles.append(dstdir) continue if not dstdir.endswith("/"): dstdir = dstdir + "/" shareddirs.append(dstdir) # Check the file list for conflicts against files which already exist overlap_allowed = (d.getVar("SSTATE_ALLOW_OVERLAP_FILES") or "").split() match = [] for f in sharedfiles: if os.path.exists(f): f = os.path.normpath(f) realmatch = True for w in overlap_allowed: w = os.path.normpath(w) if f.startswith(w): realmatch = False break if realmatch: match.append(f) sstate_search_cmd = "grep -rlF '%s' %s --exclude=index-* | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}")) search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] if search_output: match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip()) else: match.append(" (not matched to any task)") if match: bb.fatal("Recipe %s is trying to install files into a shared " \ "area when those files already exist. The files and the manifests listing " \ "them are:\n %s\n" "Please adjust the recipes so only one recipe provides a given file. " % \ (d.getVar('PN'), "\n ".join(match))) if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"): sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd") sharedfiles.append(ss['fixmedir'] + "/fixmepath") # Write out the manifest f = open(manifest, "w") for file in sharedfiles: f.write(file + "\n") # We want to ensure that directories appear at the end of the manifest # so that when we test to see if they should be deleted any contents # added by the task will have been removed first. dirs = sorted(shareddirs, key=len) # Must remove children first, which will have a longer path than the parent for di in reversed(dirs): f.write(di + "\n") f.close() # Append to the list of manifests for this PACKAGE_ARCH i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}") l = bb.utils.lockfile(i + ".lock") filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n" manifests = [] if os.path.exists(i): with open(i, "r") as f: manifests = f.readlines() # We append new entries, we don't remove older entries which may have the same # manifest name but different versions from stamp/workdir. See below. if filedata not in manifests: with open(i, "a+") as f: f.write(filedata) bb.utils.unlockfile(l) # Run the actual file install for state in ss['dirs']: if os.path.exists(state[1]): oe.path.copyhardlinktree(state[1], state[2]) for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split(): # All hooks should run in the SSTATE_INSTDIR bb.build.exec_func(postinst, d, (sstateinst,)) for lock in locks: bb.utils.unlockfile(lock) def sstate_clean_manifest(manifest, d, canrace=False, prefix=None): import oe.path mfile = open(manifest) entries = mfile.readlines() mfile.close() for entry in entries: entry = entry.strip() if prefix and not entry.startswith("/"): entry = prefix + "/" + entry bb.debug(2, "Removing manifest: %s" % entry) # We can race against another package populating directories as we're removing them # so we ignore errors here. try: if entry.endswith("/"): if os.path.islink(entry[:-1]): os.remove(entry[:-1]) elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace: # Removing directories whilst builds are in progress exposes a race. Only # do it in contexts where it is safe to do so. os.rmdir(entry[:-1]) else: os.remove(entry) except OSError: pass postrm = manifest + ".postrm" if os.path.exists(manifest + ".postrm"): import subprocess os.chmod(postrm, 0o755) subprocess.check_call(postrm, shell=True) oe.path.remove(postrm) oe.path.remove(manifest)