KiBot/kibot/kicad/config.py

667 lines
26 KiB
Python

# -*- coding: utf-8 -*-
# Copyright (c) 2020-2023 Salvador E. Tropea
# Copyright (c) 2020-2023 Instituto Nacional de Tecnología Industrial
# License: GPL-3.0
# Project: KiBot (formerly KiPlot)
"""
KiCad configuration classes.
Reads the KiCad's configuration files.
In particular:
- kicad_common to know about the 'environment' variables
- The `sym-lib-table` files to map library aliases
Notes about coverage:
I'm excluding all the Darwin and Windows code from coverage.
I'm not even sure the values are correct.
"""
import csv
from glob import glob
from io import StringIO
import json
import os
import platform
import re
from shutil import copy2
import sys
import sysconfig
from ..error import KiPlotConfigurationError
from ..gs import GS
from .. import log
from ..misc import (W_NOCONFIG, W_NOKIENV, W_NOLIBS, W_NODEFSYMLIB, MISSING_WKS, W_MAXDEPTH, W_3DRESVER, W_LIBTVERSION,
W_LIBTUNK)
from .sexpdata import load, SExpData
from .sexp_helpers import _check_is_symbol_list, _check_integer, _check_relaxed
# Check python version to determine which version of ConfirParser to import
if sys.version_info.major >= 3:
import configparser as ConfigParser
else: # pragma: no cover (Py2)
# For future Python 2 support
import ConfigParser
logger = log.get_logger()
SYM_LIB_TABLE = 'sym-lib-table'
FP_LIB_TABLE = 'fp-lib-table'
KICAD_COMMON = 'kicad_common'
SUP_VERSION = 7
reported = set()
class KiConfError(Exception):
def __init__(self, msg, file, lnum, line):
super().__init__()
self.line = lnum
self.file = file
self.msg = msg
self.code = line
def un_quote(val):
""" Remove optional quotes """
if val[0] == '"':
val.replace('\"', '"')
val = val[1:-1]
return val
def parse_len_str(val):
if ':' not in val:
return val
pos = val.index(':')
try:
c = int(val[:pos])
except ValueError:
c = None
logger.error('Malformed 3D alias entry: '+val)
value = val[pos+1:]
if c is not None and c != len(value):
logger.error('3D alias entry error, expected len {}, but found {}'.format(c, len(value)))
return value
def expand_env(val, env, extra_env, used_extra=None):
""" Expand KiCad environment variables """
if used_extra is None:
used_extra = [False]
used_extra[0] = False
success = replaced = True
depth = 0
ori_val = val
while success and replaced and depth < GS.MAXDEPTH:
replaced = False
depth += 1
if depth == GS.MAXDEPTH:
logger.warning(W_MAXDEPTH+'Too much nested variables replacements, possible loop ({})'.format(ori_val))
success = False
for var in re.findall(r'\$\{(\S+?)\}', val):
to_replace = '${'+var+'}'
if var in env:
val = val.replace(to_replace, env[var])
replaced = True
elif var in extra_env:
val = val.replace(to_replace, extra_env[var])
used_extra[0] = True
replaced = True
elif GS.global_use_os_env_for_expand and var in os.environ:
val = val.replace(to_replace, os.environ[var])
used_extra[0] = True
replaced = True
else:
success = False
# Note: We can't expand NET_NAME(n)
if var not in reported and not var.startswith('NET_NAME('):
logger.error('Unable to expand `{}` in `{}`'.format(var, val))
reported.add(var)
return val
class LibAlias(object):
""" An entry for the symbol libs table """
def __init__(self):
super().__init__()
self.name = None
self.legacy = True
self.uri = None
self.options = None
self.descr = None
@staticmethod
def parse(items, env, extra_env):
s = LibAlias()
for i in items[1:]:
i_type = _check_is_symbol_list(i)
if i_type == 'name':
s.name = _check_relaxed(i, 1, i_type)
elif i_type == 'type':
s.type = _check_relaxed(i, 1, i_type)
elif i_type == 'uri':
s.uri = os.path.abspath(expand_env(_check_relaxed(i, 1, i_type), env, extra_env))
elif i_type == 'options':
s.options = _check_relaxed(i, 1, i_type)
elif i_type == 'descr':
s.descr = _check_relaxed(i, 1, i_type)
else:
logger.warning(W_LIBTUNK+'Unknown lib table attribute `{}`'.format(i))
return s
def __str__(self):
if not self.name:
return 'unnamed LibAlias'
return '`'+self.name+'` -> `'+self.uri+'`'
class KiConf(object):
""" Class to load and hold all KiCad configuration """
loaded = False
config_dir = None
dirname = None
sym_lib_dir = None
template_dir = None
footprint_dir = None
models_3d_dir = None
party_3rd_dir = None
kicad_env = {}
lib_aliases = None
fp_aliases = None
aliases_3D = {}
def __init__(self):
raise AssertionError("KiConf is fully static, no instances allowed")
def init(fname):
""" fname is the base project name, any extension is allowed.
So it can be the main schematic, the PCB or the project. """
if KiConf.loaded:
return
KiConf.dirname = os.path.dirname(fname)
KiConf.kicad_env['KIPRJMOD'] = KiConf.dirname
KiConf.load_kicad_common()
KiConf.load_3d_aliases()
KiConf.loaded = True
# Loaded on demand, here to debug
# KiConf.get_sym_lib_aliases()
# logger.error(KiConf.lib_aliases)
# KiConf.get_fp_lib_aliases()
# logger.error(KiConf.fp_aliases)
def find_kicad_common():
""" Looks for kicad_common config file.
Returns its name or None. """
cfg = ''
if GS.kicad_conf_path:
cfg = os.path.join(GS.kicad_conf_path, KICAD_COMMON)
if GS.ki6:
cfg += '.json'
if os.path.isfile(cfg):
return cfg
logger.warning(W_NOCONFIG + 'Unable to find KiCad configuration file ({})'.format(cfg))
return None
def _guess_kicad_data_dir(data_dir):
""" Tries to figure out where libraries are.
Only used if we failed to find the kicad_common file.
On modern KiCad (6+) this is always used because KiCad doesn't store the path in kicad_common,
unless modified by the user. """
# Give priority to the KICAD_PATH environment variable
kpath = os.environ.get('KICAD_PATH')
if kpath and os.path.isdir(kpath):
dir = os.path.join(kpath, data_dir)
if os.path.isdir(dir):
return dir
# Try to guess according to the OS
system = platform.system()
share = os.path.join('share', GS.kicad_dir, data_dir)
if system == 'Linux':
scheme_names = sysconfig.get_scheme_names()
# Try in local dir
if 'posix_user' in scheme_names:
dir = os.path.join(sysconfig.get_path('data', 'posix_user'), share)
if os.path.isdir(dir):
return dir
# Try at system level
if 'posix_prefix' in scheme_names:
dir = os.path.join(sysconfig.get_path('data', 'posix_prefix'), share)
if os.path.isdir(dir):
return dir
elif system == 'Darwin': # pragma: no cover (Darwin)
app_data = os.path.join('Library', 'Application Support', GS.kicad_dir, data_dir)
home = os.environ.get('HOME')
if home:
dir = os.path.join(home, app_data)
if os.path.isdir(dir):
return dir
dir = os.path.join('/', app_data)
if os.path.isdir(dir):
return dir
elif system == 'Windows': # pragma: no cover (Windows)
dir = os.path.join('C:', 'Program Files', 'KiCad', share)
if os.path.isdir(dir):
return dir
dir = os.path.join('C:', 'KiCad', share)
if os.path.isdir(dir):
return dir
username = os.environ.get('username')
dir = os.path.join('C:', 'Users', username, 'Documents', 'KiCad', data_dir)
if os.path.isdir(dir):
return dir
return None
def guess_symbol_dir():
if GS.ki5:
order = ['library', 'symbols']
else:
order = ['symbols', 'library']
guess = KiConf._guess_kicad_data_dir(order[0])
if guess is None:
guess = KiConf._guess_kicad_data_dir(order[1])
return guess
def guess_footprint_dir():
if GS.ki5:
order = ['modules', 'footprints']
else:
order = ['footprints', 'modules']
guess = KiConf._guess_kicad_data_dir(order[0])
if guess is None:
guess = KiConf._guess_kicad_data_dir(order[1])
return guess
def guess_template_dir():
return KiConf._guess_kicad_data_dir('template')
def guess_3d_dir():
modules3d = os.path.join('modules', 'packages3d')
if GS.ki5:
order = [modules3d, '3dmodels']
else:
order = ['3dmodels', modules3d]
guess = KiConf._guess_kicad_data_dir(order[0])
if guess is None:
guess = KiConf._guess_kicad_data_dir(order[1])
return guess
def guess_user_template_dir():
system = platform.system()
if system == 'Linux':
home = os.environ.get('HOME')
if home is None:
return None
if GS.ki6:
name = os.path.join(home, '.local', 'share', 'kicad', str(GS.kicad_version_major)+'.0', 'template')
if os.path.isdir(name):
return name
name = os.path.join(home, 'kicad', 'template')
if os.path.isdir(name):
return name
return None
elif system == 'Darwin': # pragma: no cover (Darwin)
home = os.environ.get('HOME')
if home is None:
return None
name = os.path.join(home, 'Documents', 'kicad', 'template')
if os.path.isdir(name):
return name
return None
elif system == 'Windows': # pragma: no cover (Windows)
username = os.environ.get('username')
dir = os.path.join('C:', 'Documents and Settings', username, 'My Documents', 'kicad', 'template')
if os.path.isdir(dir):
return dir
dir = os.path.join('C:', 'Users', username, 'Documents', 'kicad', 'template')
if os.path.isdir(dir):
return dir
return None
def guess_3rd_dir():
home = os.environ.get('HOME')
if home is None:
return None
return os.path.join(home, '.local', 'share', 'kicad', '6.0', '3rdparty')
def load_ki5_env(cfg):
""" Environment vars from KiCad 5 configuration """
# Load the "environment variables"
with open(cfg, 'rt') as f:
buf = f.read()
io_buf = StringIO('[Default]\n'+buf)
cf = ConfigParser.RawConfigParser(allow_no_value=True)
cf.optionxform = str
if sys.version_info.major >= 3:
cf.read_file(io_buf, cfg)
else: # pragma: no cover (Py2)
cf.readfp(io_buf, cfg)
if 'EnvironmentVariables' not in cf.sections():
logger.warning(W_NOKIENV + 'KiCad config without EnvironmentVariables section')
else:
for k, v in cf.items('EnvironmentVariables'):
if GS.debug_level > 1:
logger.debug('- KiCad var: {}="{}"'.format(k, v))
KiConf.kicad_env[k] = v
def load_ki6_env(cfg):
""" Environment vars from KiCad 6 configuration """
# Notes about KiCad 6 environment vars:
# 1) KiCad has some hardcoded internal values
# 2) Only the values that the user modifies are stored in the config file
# 3) Any value defined in the environment will have priority over internal definitions
# It means that we currently don't know the real value unless the user modifies it or defines in the env
with open(cfg, 'rt') as f:
data = json.load(f)
if "environment" in data and 'vars' in data['environment'] and (data['environment']['vars'] is not None):
for k, v in data['environment']['vars'].items():
if GS.debug_level > 1:
logger.debug('- KiCad var: {}="{}"'.format(k, v))
KiConf.kicad_env[k] = v
else:
logger.warning(W_NOKIENV + 'KiCad config without environment.vars section')
def _look_env_var(base_name, old, only_old, ki6_diff, no_dir):
""" Looks for a KiCad variable definition.
First in the OS environment, then in the config.
Sync the other source. """
if not no_dir:
base_name += '_DIR'
names = []
if GS.ki6 and ki6_diff:
# KiCad 6 specific name goes first when using KiCad 6
for n in reversed(range(6, GS.kicad_version_major+1)):
names.append('KICAD{}_{}'.format(n, base_name))
# KiCad 5 names, allowed even when using KiCad 6
if not only_old:
# A KICAD_* is valid
names.append('KICAD_'+base_name)
if old:
# A specific legacy name
names.append(old)
ret_val = None
# Look for the above names
# The first we find is the one we use, but we export all the ones we found
for n in names:
val = None
# OS environment has more priority
if n in os.environ:
val = os.environ[n]
KiConf.kicad_env[n] = val
logger.debug('Using {}="{}" (from environment)'.format(n, val))
# Then the file
elif n in KiConf.kicad_env:
val = KiConf.kicad_env[n]
os.environ[n] = val
logger.debug('Using {}="{}" (from KiCad config)'.format(n, val))
if val is not None and ret_val is None:
ret_val = val
# Make sure all the versions point to the same place
# It helps if we are using KiCad 6 but have the KiCad 5 names defined,
# KiCad will use them, but any mention to the KiCad 6 version won't be
# valid for KiBot unless we explicitly define it.
if ret_val is not None:
for n in names:
if n not in os.environ or n not in KiConf.kicad_env:
os.environ[n] = ret_val
KiConf.kicad_env[n] = ret_val
return ret_val
def _set_env_var(base_name, val, ki6_diff, no_dir):
""" Sets the environment and the internal list """
if not no_dir:
base_name += '_DIR'
if GS.ki6 and ki6_diff:
name = 'KICAD{}_{}'.format(GS.kicad_version_major, base_name)
else:
name = 'KICAD_'+base_name
KiConf.kicad_env[name] = val
os.environ[name] = val
logger.debug('Using {}="{}" (guessed)'.format(name, val))
def _solve_var(name, member, desc, guesser, old=None, only_old=False, ki6_diff=True, only_k6=False, no_dir=False):
if only_k6 and GS.ki5:
return
val = KiConf._look_env_var(name, old, only_old, ki6_diff, no_dir)
if val is not None:
setattr(KiConf, member, val)
else:
val = guesser()
if val:
setattr(KiConf, member, val)
KiConf._set_env_var(name, val, ki6_diff, no_dir)
if old:
KiConf.kicad_env[old] = val
os.environ[old] = val
else:
logger.warning(W_NOLIBS + 'Unable to find KiCad '+desc)
return val
def load_kicad_common():
# Try to figure out KiCad configuration file
cfg = KiConf.find_kicad_common()
if cfg and os.path.isfile(cfg):
# Get the environment variables
logger.debug('Reading KiCad config from `{}`'.format(cfg))
KiConf.config_dir = os.path.dirname(cfg)
if GS.ki5:
# All environment vars should be here
KiConf.load_ki5_env(cfg)
else:
# Only the user defined environment vars are here
KiConf.load_ki6_env(cfg)
# Now make sure we have all the needed variables
KiConf._solve_var('SYMBOL', 'sym_lib_dir', 'libraries', KiConf.guess_symbol_dir)
KiConf._solve_var('TEMPLATE', 'template_dir', 'templates', KiConf.guess_template_dir)
KiConf._solve_var('USER_TEMPLATE', 'user_template_dir', 'user templates', KiConf.guess_user_template_dir,
ki6_diff=False)
KiConf._solve_var('FOOTPRINT', 'footprint_dir', 'footprints', KiConf.guess_footprint_dir, old='KISYSMOD')
KiConf._solve_var('3DMODEL', 'models_3d_dir', '3D models', KiConf.guess_3d_dir, old='KISYS3DMOD', only_old=True)
KiConf._solve_var('3RD_PARTY', 'party_3rd_dir', '3rd party', KiConf.guess_3rd_dir, only_k6=True, no_dir=True)
# Export the rest. KiCad2Step needs them
to_add = {}
for k, v in KiConf.kicad_env.items():
if k not in os.environ:
os.environ[k] = v
logger.debug('Exporting {}="{}"'.format(k, v))
m = re.match(r'KICAD(\d+)_(.*)', k)
if m:
for n in range(6, GS.kicad_version_major+1):
kv = 'KICAD'+str(n)+'_'+m.group(2)
if kv not in os.environ and kv not in KiConf.kicad_env and kv not in to_add:
os.environ[kv] = v
to_add[kv] = v
logger.debug('Also exporting {}="{}"'.format(kv, v))
KiConf.kicad_env.update(to_add)
def load_lib_aliases(fname, lib_aliases):
if not os.path.isfile(fname):
return False
logger.debug('Loading symbols lib table `{}`'.format(fname))
version = 0
with open(fname, 'rt') as f:
error = None
try:
table = load(f)[0]
except SExpData as e:
error = str(e)
if error:
raise KiPlotConfigurationError('Error loading `{}`: {}'.format(fname, error))
if not isinstance(table, list) or (table[0].value() != 'sym_lib_table' and table[0].value() != 'fp_lib_table'):
raise KiPlotConfigurationError('Error loading `{}`: not a library table'.format(fname))
for e in table[1:]:
e_type = _check_is_symbol_list(e)
if e_type == 'version':
version = _check_integer(e, 1, e_type)
if version > SUP_VERSION:
logger.warning(W_LIBTVERSION+"Unsupported lib table version, loading could fail")
elif e_type == 'lib':
alias = LibAlias.parse(e, KiConf.kicad_env, {})
if GS.debug_level > 1:
logger.debug('- Adding lib alias '+str(alias))
lib_aliases[alias.name] = alias
else:
logger.warning(W_LIBTUNK+"Unknown lib table entry `{}`".format(e_type))
return True
def load_all_lib_aliases(table_name, sys_dir, pattern):
# Load the default symbol libs table.
# This is the list of libraries enabled by the user.
loaded = False
lib_aliases = {}
if KiConf.config_dir:
conf_dir = KiConf.config_dir
if 'KICAD_CONFIG_HOME' in KiConf.kicad_env:
# KiCad 5 unintentionally allows it, is a bug, and won't be fixed:
# https://forum.kicad.info/t/kicad-config-home-inconsistencies-and-detail/26875
conf_dir = KiConf.kicad_env['KICAD_CONFIG_HOME']
logger.debug('Redirecting symbols lib table to '+conf_dir)
loaded = KiConf.load_lib_aliases(os.path.join(conf_dir, table_name), lib_aliases)
if not loaded and 'KICAD_TEMPLATE_DIR' in KiConf.kicad_env:
loaded = KiConf.load_lib_aliases(os.path.join(KiConf.kicad_env['KICAD_TEMPLATE_DIR'], table_name), lib_aliases)
if not loaded:
logger.warning(W_NODEFSYMLIB + 'Missing default symbol library table')
# No default symbol libs table, try to create one
if KiConf.sym_lib_dir:
for f in glob(os.path.join(sys_dir, pattern)):
alias = LibAlias()
alias.name = os.path.splitext(os.path.basename(f))[0]
alias.uri = f
if GS.debug_level > 1:
logger.debug('Detected lib alias '+str(alias))
lib_aliases[alias.name] = alias
# Load the project's table
KiConf.load_lib_aliases(os.path.join(KiConf.dirname, table_name), lib_aliases)
return lib_aliases
def get_sym_lib_aliases(fname=None):
if KiConf.lib_aliases is None:
if fname is None:
fname = GS.sch_file
KiConf.init(fname)
pattern = '*.kicad_sym' if GS.ki6 else '*.lib'
KiConf.lib_aliases = KiConf.load_all_lib_aliases(SYM_LIB_TABLE, KiConf.sym_lib_dir, pattern)
return KiConf.lib_aliases
def get_fp_lib_aliases(fname=None):
if KiConf.fp_aliases is None:
if fname is None:
fname = GS.pcb_file
KiConf.init(fname)
KiConf.fp_aliases = KiConf.load_all_lib_aliases(FP_LIB_TABLE, KiConf.footprint_dir, '*.pretty')
return KiConf.fp_aliases
def load_3d_aliases():
if not KiConf.config_dir:
return
fname = os.path.join(KiConf.config_dir, '3d', '3Dresolver.cfg')
if not os.path.isfile(fname):
logger.debug('No 3D aliases ({})'.format(fname))
return
logger.debug('Loading 3D aliases from '+fname)
with open(fname) as f:
reader = csv.reader(f)
head = next(reader)
if len(head) != 1 or head[0] != '#V1':
logger.warning(W_3DRESVER, 'Unsupported 3D resolver version ({})'.format(head))
for r in reader:
if len(r) != 3:
logger.error("3D resolver doesn't contain three values ({})".format(r))
continue
name = parse_len_str(r[0])
value = parse_len_str(r[1])
# Discard the comment (2)
logger.debugl(1, '- {}={}'.format(name, value))
KiConf.aliases_3D[name] = value
logger.debugl(1, 'Finished loading 3D aliases')
def fix_page_layout_k6_key(key, data, dest_dir):
if key in data:
section = data[key]
pl = section.get('page_layout_descr_file', None)
if pl:
fname = KiConf.expand_env(pl)
if os.path.isfile(fname):
dest = os.path.join(dest_dir, key+'.kicad_wks')
logger.debug('Copying {} -> {}'.format(fname, dest))
copy2(fname, dest)
data[key]['page_layout_descr_file'] = dest
return dest
else:
logger.error('Missing page layout file: '+fname)
exit(MISSING_WKS)
return None
def fix_page_layout_k6(project, dry):
# Get the current definitions
dest_dir = os.path.dirname(project)
with open(project, 'rt') as f:
pro_text = f.read()
data = json.loads(pro_text)
layouts = [None, None]
if not dry:
layouts[1] = KiConf.fix_page_layout_k6_key('pcbnew', data, dest_dir)
layouts[0] = KiConf.fix_page_layout_k6_key('schematic', data, dest_dir)
with open(project, 'wt') as f:
f.write(json.dumps(data, sort_keys=True, indent=2))
else:
aux = data.get('schematic', None)
if aux:
layouts[0] = KiConf.expand_env(aux.get('page_layout_descr_file', None))
aux = data.get('pcbnew', None)
if aux:
layouts[1] = KiConf.expand_env(aux.get('page_layout_descr_file', None))
return layouts
def fix_page_layout_k5(project, dry):
order = 1
dest_dir = os.path.dirname(project)
with open(project, 'rt') as f:
lns = f.readlines()
is_pcb_new = False
layouts = [None, None]
for c, line in enumerate(lns):
if line.startswith('[pcbnew]'):
is_pcb_new = True
if line.startswith('[schematic'):
is_pcb_new = False
if line.startswith('PageLayoutDescrFile='):
fname = line[20:].strip()
if fname:
fname = KiConf.expand_env(fname)
if os.path.isfile(fname):
dest = os.path.join(dest_dir, str(order)+'.kicad_wks')
if not dry:
copy2(fname, dest)
layouts[is_pcb_new] = dest
else:
layouts[is_pcb_new] = fname
order = order+1
else:
logger.error('Missing page layout file: '+fname)
exit(MISSING_WKS)
else:
dest = ''
lns[c] = 'PageLayoutDescrFile='+dest+'\n'
if not dry:
with open(project, 'wt') as f:
lns = f.writelines(lns)
return layouts
def fix_page_layout(project, dry=False):
if not project:
return None, None
KiConf.init(GS.pcb_file)
if GS.ki5:
return KiConf.fix_page_layout_k5(project, dry)
return KiConf.fix_page_layout_k6(project, dry)
def expand_env(name, used_extra=None, ref_dir=None):
if used_extra is None:
used_extra = [False]
if not name:
return name
expanded = expand_env(un_quote(name), KiConf.kicad_env, GS.load_pro_variables(), used_extra)
# Don't try to get the absolute path for something that starts with a variable that we couldn't expand
if ref_dir is None:
ref_dir = os.getcwd()
return expanded if expanded.startswith('${') else os.path.normpath(os.path.join(ref_dir, expanded))
# Avoid circular inclusion
GS.fix_page_layout = KiConf.fix_page_layout