#
|
# BitBake Toaster Implementation
|
#
|
# Copyright (C) 2018 Wind River Systems
|
#
|
# SPDX-License-Identifier: GPL-2.0-only
|
#
|
|
# buildimport: import a project for project specific configuration
|
#
|
# Usage:
|
# (a) Set up Toaster environent
|
#
|
# (b) Call buildimport
|
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
|
# --name=$PROJECTNAME \
|
# --path=$BUILD_DIRECTORY \
|
# --callback="$CALLBACK_SCRIPT" \
|
# --command="configure|reconfigure|import"
|
#
|
# (c) Return is "|Default_image=%s|Project_id=%d"
|
#
|
# (d) Open Toaster to this project using for example:
|
# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
|
#
|
# (e) To delete a project:
|
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
|
# --name=$PROJECTNAME --delete-project
|
#
|
|
|
# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
|
|
from django.core.management.base import BaseCommand
|
from orm.models import Project, Release, ProjectVariable
|
from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
|
from toastergui.api import scan_layer_content
|
|
import os
|
import re
|
import os.path
|
import subprocess
|
import shutil
|
|
# Toaster variable section delimiters
|
TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
|
TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
|
|
# quick development/debugging support
|
verbose = 2
|
def _log(msg):
|
if 1 == verbose:
|
print(msg)
|
elif 2 == verbose:
|
f1=open('/tmp/toaster.log', 'a')
|
f1.write("|" + msg + "|\n" )
|
f1.close()
|
|
|
__config_regexp__ = re.compile( r"""
|
^
|
(?P<exp>export\s+)?
|
(?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
|
(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
|
|
\s* (
|
(?P<colon>:=) |
|
(?P<lazyques>\?\?=) |
|
(?P<ques>\?=) |
|
(?P<append>\+=) |
|
(?P<prepend>=\+) |
|
(?P<predot>=\.) |
|
(?P<postdot>\.=) |
|
=
|
) \s*
|
|
(?!'[^']*'[^']*'$)
|
(?!\"[^\"]*\"[^\"]*\"$)
|
(?P<apo>['\"])
|
(?P<value>.*)
|
(?P=apo)
|
$
|
""", re.X)
|
|
class Command(BaseCommand):
|
args = "<name> <path> <release>"
|
help = "Import a command line build directory"
|
vars = {}
|
toaster_vars = {}
|
|
def add_arguments(self, parser):
|
parser.add_argument(
|
'--name', dest='name', required=True,
|
help='name of the project',
|
)
|
parser.add_argument(
|
'--path', dest='path', required=True,
|
help='path to the project',
|
)
|
parser.add_argument(
|
'--release', dest='release', required=False,
|
help='release for the project',
|
)
|
parser.add_argument(
|
'--callback', dest='callback', required=False,
|
help='callback for project config update',
|
)
|
parser.add_argument(
|
'--delete-project', dest='delete_project', required=False,
|
help='delete this project from the database',
|
)
|
parser.add_argument(
|
'--command', dest='command', required=False,
|
help='command (configure,reconfigure,import)',
|
)
|
|
def get_var(self, varname):
|
value = self.vars.get(varname, '')
|
if value:
|
varrefs = re.findall('\${([^}]*)}', value)
|
for ref in varrefs:
|
if ref in self.vars:
|
value = value.replace('${%s}' % ref, self.vars[ref])
|
return value
|
|
# Extract the bb variables from a conf file
|
def scan_conf(self,fn):
|
vars = self.vars
|
toaster_vars = self.toaster_vars
|
|
#_log("scan_conf:%s" % fn)
|
if not os.path.isfile(fn):
|
return
|
f = open(fn, 'r')
|
|
#statements = ast.StatementGroup()
|
lineno = 0
|
is_toaster_section = False
|
while True:
|
lineno = lineno + 1
|
s = f.readline()
|
if not s:
|
break
|
w = s.strip()
|
# skip empty lines
|
if not w:
|
continue
|
# evaluate Toaster sections
|
if w.startswith(TOASTER_PROLOG):
|
is_toaster_section = True
|
continue
|
if w.startswith(TOASTER_EPILOG):
|
is_toaster_section = False
|
continue
|
s = s.rstrip()
|
while s[-1] == '\\':
|
s2 = f.readline().strip()
|
lineno = lineno + 1
|
if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
|
echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
|
s = s[:-1] + s2
|
# skip comments
|
if s[0] == '#':
|
continue
|
# process the line for just assignments
|
m = __config_regexp__.match(s)
|
if m:
|
groupd = m.groupdict()
|
var = groupd['var']
|
value = groupd['value']
|
|
if groupd['lazyques']:
|
if not var in vars:
|
vars[var] = value
|
continue
|
if groupd['ques']:
|
if not var in vars:
|
vars[var] = value
|
continue
|
# preset empty blank for remaining operators
|
if not var in vars:
|
vars[var] = ''
|
if groupd['append']:
|
vars[var] += value
|
elif groupd['prepend']:
|
vars[var] = "%s%s" % (value,vars[var])
|
elif groupd['predot']:
|
vars[var] = "%s %s" % (value,vars[var])
|
elif groupd['postdot']:
|
vars[var] = "%s %s" % (vars[var],value)
|
else:
|
vars[var] = "%s" % (value)
|
# capture vars in a Toaster section
|
if is_toaster_section:
|
toaster_vars[var] = vars[var]
|
|
# DONE WITH PARSING
|
f.close()
|
self.vars = vars
|
self.toaster_vars = toaster_vars
|
|
# Update the scanned project variables
|
def update_project_vars(self,project,name):
|
pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
|
if (not name in self.vars.keys()) or (not self.vars[name]):
|
self.vars[name] = pv.value
|
else:
|
if pv.value != self.vars[name]:
|
pv.value = self.vars[name]
|
pv.save()
|
|
# Find the git version of the installation
|
def find_layer_dir_version(self,path):
|
# * rocko ...
|
|
install_version = ''
|
cwd = os.getcwd()
|
os.chdir(path)
|
p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
|
stderr=subprocess.PIPE)
|
out, err = p.communicate()
|
out = out.decode("utf-8")
|
for branch in out.split('\n'):
|
if ('*' == branch[0:1]) and ('no branch' not in branch):
|
install_version = re.sub(' .*','',branch[2:])
|
break
|
if 'remotes/m/master' in branch:
|
install_version = re.sub('.*base/','',branch)
|
break
|
os.chdir(cwd)
|
return install_version
|
|
# Compute table of the installation's registered layer versions (branch or commit)
|
def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
|
lv_dict = {}
|
layer_versions = Layer_Version.objects.all()
|
for lv in layer_versions:
|
layer = Layer.objects.filter(pk=lv.layer.pk)[0]
|
if layer.vcs_url:
|
url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
|
else:
|
url_short = ''
|
# register the core, branch, and the version variations
|
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
|
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
|
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
|
#_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
|
return lv_dict
|
|
# Apply table of all layer versions
|
def extract_bblayers(self):
|
# set up the constants
|
bblayer_str = self.get_var('BBLAYERS')
|
TOASTER_DIR = os.environ.get('TOASTER_DIR')
|
INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
|
TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
|
INSTALL_URL_PREFIX = ''
|
layers = Layer.objects.filter(name='openembedded-core')
|
for layer in layers:
|
if layer.vcs_url:
|
INSTALL_URL_PREFIX = layer.vcs_url
|
break
|
INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
|
INSTALL_VERSION_DIR = TOASTER_DIR
|
INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
|
INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
|
INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
|
|
# get the set of available layer:layer_versions
|
lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
|
|
# compute the layer matches
|
layers_list = []
|
for line in bblayer_str.split(' '):
|
if not line:
|
continue
|
if line.endswith('/local'):
|
continue
|
|
# isolate the repo
|
layer_path = line
|
line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
|
|
# isolate the sub-path
|
path_index = line.rfind('/')
|
if path_index > 0:
|
sub_path = line[path_index+1:]
|
line = line[0:path_index]
|
else:
|
sub_path = ''
|
|
# isolate the version
|
if TOASTER_CLONE_PREFIX in layer_path:
|
is_toaster_clone = True
|
# extract version from name syntax
|
version_index = line.find('_')
|
if version_index > 0:
|
version = line[version_index+1:]
|
line = line[0:version_index]
|
else:
|
version = ''
|
_log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
|
else:
|
is_toaster_clone = False
|
# version is from the installation
|
version = self.find_layer_dir_version(layer_path)
|
_log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
|
|
# capture the layer information into layers_list
|
layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
|
return layers_list,lv_dict
|
|
#
|
def find_import_release(self,layers_list,lv_dict,default_release):
|
# poky,meta,rocko => 4;openembedded-core
|
release = default_release
|
for line,path,version,layer_path,is_toaster_clone in layers_list:
|
key = "%s,%s,%s" % (line,path,version)
|
if key in lv_dict:
|
lv_id = lv_dict[key]
|
if 'openembedded-core' == lv_id[1]:
|
_log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
|
# only versions in Toaster managed layers are accepted
|
if not is_toaster_clone:
|
break
|
try:
|
release = Release.objects.get(name=version)
|
except:
|
pass
|
break
|
_log("Find_import_release:RELEASE=%s" % release.name)
|
return release
|
|
# Apply the found conf layers
|
def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
|
for line,path,version,layer_path,is_toaster_clone in layers_list:
|
# Assert release promote if present
|
if release:
|
version = release
|
# try to match the key to a layer_version
|
key = "%s,%s,%s" % (line,path,version)
|
key_short = "%s,%s,%s" % (line,path,'')
|
lv_id = ''
|
if key in lv_dict:
|
lv_id = lv_dict[key]
|
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
|
pl,created = ProjectLayer.objects.get_or_create(project=project,
|
layercommit=lv)
|
pl.optional=False
|
pl.save()
|
_log(" %s => %s;%s" % (key,lv_id[0],lv_id[1]))
|
elif key_short in lv_dict:
|
lv_id = lv_dict[key_short]
|
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
|
pl,created = ProjectLayer.objects.get_or_create(project=project,
|
layercommit=lv)
|
pl.optional=False
|
pl.save()
|
_log(" %s ?> %s" % (key,lv_dict[key_short]))
|
else:
|
_log("%s <= %s" % (key,layer_path))
|
found = False
|
# does local layer already exist in this project?
|
try:
|
for pl in ProjectLayer.objects.filter(project=project):
|
if pl.layercommit.layer.local_source_dir == layer_path:
|
found = True
|
_log(" Project Local Layer found!")
|
except Exception as e:
|
_log("ERROR: Local Layer '%s'" % e)
|
pass
|
|
if not found:
|
# Does Layer name+path already exist?
|
try:
|
layer_name_base = os.path.basename(layer_path)
|
_log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
|
layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
|
# Found! Attach layer_version and ProjectLayer
|
layer_version = Layer_Version.objects.create(
|
layer=layer,
|
project=project,
|
layer_source=LayerSource.TYPE_IMPORTED)
|
layer_version.save()
|
pl,created = ProjectLayer.objects.get_or_create(project=project,
|
layercommit=layer_version)
|
pl.optional=False
|
pl.save()
|
found = True
|
# add layer contents to this layer version
|
scan_layer_content(layer,layer_version)
|
_log(" Parent Local Layer found in db!")
|
except Exception as e:
|
_log("Layer_exists_test_failed: Local Layer '%s'" % e)
|
pass
|
|
if not found:
|
# Insure that layer path exists, in case of user typo
|
if not os.path.isdir(layer_path):
|
_log("ERROR:Layer path '%s' not found" % layer_path)
|
continue
|
# Add layer to db and attach project to it
|
layer_name_base = os.path.basename(layer_path)
|
# generate a unique layer name
|
layer_name_matches = {}
|
for layer in Layer.objects.filter(name__contains=layer_name_base):
|
layer_name_matches[layer.name] = '1'
|
layer_name_idx = 0
|
layer_name_test = layer_name_base
|
while layer_name_test in layer_name_matches.keys():
|
layer_name_idx += 1
|
layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
|
# create the layer and layer_verion objects
|
layer = Layer.objects.create(name=layer_name_test)
|
layer.local_source_dir = layer_path
|
layer_version = Layer_Version.objects.create(
|
layer=layer,
|
project=project,
|
layer_source=LayerSource.TYPE_IMPORTED)
|
layer.save()
|
layer_version.save()
|
pl,created = ProjectLayer.objects.get_or_create(project=project,
|
layercommit=layer_version)
|
pl.optional=False
|
pl.save()
|
# register the layer's content
|
_log(" Local Layer Add content")
|
scan_layer_content(layer,layer_version)
|
_log(" Local Layer Added '%s'!" % layer_name_test)
|
|
# Scan the project's conf files (if any)
|
def scan_conf_variables(self,project_path):
|
self.vars['TOPDIR'] = project_path
|
# scan the project's settings, add any new layers or variables
|
if os.path.isfile("%s/conf/local.conf" % project_path):
|
self.scan_conf("%s/conf/local.conf" % project_path)
|
self.scan_conf("%s/conf/bblayers.conf" % project_path)
|
# Import then disable old style Toaster conf files (before 'merged_attr')
|
old_toaster_local = "%s/conf/toaster.conf" % project_path
|
if os.path.isfile(old_toaster_local):
|
self.scan_conf(old_toaster_local)
|
shutil.move(old_toaster_local, old_toaster_local+"_old")
|
old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
|
if os.path.isfile(old_toaster_layer):
|
self.scan_conf(old_toaster_layer)
|
shutil.move(old_toaster_layer, old_toaster_layer+"_old")
|
|
# Scan the found conf variables (if any)
|
def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
|
if self.vars:
|
# Catch vars relevant to Toaster (in case no Toaster section)
|
self.update_project_vars(project,'DISTRO')
|
self.update_project_vars(project,'MACHINE')
|
self.update_project_vars(project,'IMAGE_INSTALL:append')
|
self.update_project_vars(project,'IMAGE_FSTYPES')
|
self.update_project_vars(project,'PACKAGE_CLASSES')
|
# These vars are typically only assigned by Toaster
|
#self.update_project_vars(project,'DL_DIR')
|
#self.update_project_vars(project,'SSTATE_DIR')
|
|
# Assert found Toaster vars
|
for var in self.toaster_vars.keys():
|
pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
|
pv.value = self.toaster_vars[var]
|
_log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
|
pv.save()
|
|
# Assert found BBLAYERS
|
if 0 < verbose:
|
for pl in ProjectLayer.objects.filter(project=project):
|
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
|
print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
|
self.apply_conf_bblayers(layers_list,lv_dict,project,release)
|
if 0 < verbose:
|
for pl in ProjectLayer.objects.filter(project=project):
|
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
|
print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
|
|
def handle(self, *args, **options):
|
project_name = options['name']
|
project_path = options['path']
|
project_callback = options['callback'] if options['callback'] else ''
|
release_name = options['release'] if options['release'] else ''
|
|
#
|
# Delete project
|
#
|
|
if options['delete_project']:
|
try:
|
print("Project '%s' delete from Toaster database" % (project_name))
|
project = Project.objects.get(name=project_name)
|
# TODO: deep project delete
|
project.delete()
|
print("Project '%s' Deleted" % (project_name))
|
return
|
except Exception as e:
|
print("Project '%s' not found, not deleted (%s)" % (project_name,e))
|
return
|
|
#
|
# Create/Update/Import project
|
#
|
|
# See if project (by name) exists
|
project = None
|
try:
|
# Project already exists
|
project = Project.objects.get(name=project_name)
|
except Exception as e:
|
pass
|
|
# Find the installation's default release
|
default_release = Release.objects.get(id=1)
|
|
# SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
|
if ("reconfigure" == options['command']) and project is None:
|
options['command'] = 'import'
|
|
# 'Configure':
|
if "configure" == options['command']:
|
# Note: ignore any existing conf files
|
# create project, SANITY: reuse any project of same name
|
project = Project.objects.create_project(project_name,default_release,project)
|
|
# 'Re-configure':
|
if "reconfigure" == options['command']:
|
# Scan the directory's conf files
|
self.scan_conf_variables(project_path)
|
# Scan the layer list
|
layers_list,lv_dict = self.extract_bblayers()
|
# Apply any new layers or variables
|
self.apply_conf_variables(project,layers_list,lv_dict)
|
|
# 'Import':
|
if "import" == options['command']:
|
# Scan the directory's conf files
|
self.scan_conf_variables(project_path)
|
# Remove these Toaster controlled variables
|
for var in ('DL_DIR','SSTATE_DIR'):
|
self.vars.pop(var, None)
|
self.toaster_vars.pop(var, None)
|
# Scan the layer list
|
layers_list,lv_dict = self.extract_bblayers()
|
# Find the directory's release, and promote to default_release if local paths
|
release = self.find_import_release(layers_list,lv_dict,default_release)
|
# create project, SANITY: reuse any project of same name
|
project = Project.objects.create_project(project_name,release,project)
|
# Apply any new layers or variables
|
self.apply_conf_variables(project,layers_list,lv_dict,release)
|
# WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
|
project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
|
|
# Set up the project's meta data
|
project.builddir = project_path
|
project.merged_attr = True
|
project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
|
project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
|
if ("configure" == options['command']) or ("import" == options['command']):
|
# preset the mode and default image recipe
|
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
|
project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
|
|
# Assert any extended/custom actions or variables for new non-Toaster projects
|
if not len(self.toaster_vars):
|
pass
|
else:
|
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
|
|
# Save the updated Project
|
project.save()
|
|
_log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
|
|
if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
|
print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
|
else:
|
print("|Project_id=%d" % (project.id))
|
