[Config] More robust sym-lib-table parser

- Now loads on-the-fly (not needed for KiCad v6+)
- Uses s-expressions parser, no regex
- Can recreate the global table for KiCad v6+

kibot/kicad/config.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (c) 2020-2022 Salvador E. Tropea
+# Copyright (c) 2020-2023 Salvador E. Tropea
-# Copyright (c) 2020-2022 Instituto Nacional de Tecnología Industrial
+# Copyright (c) 2020-2023 Instituto Nacional de Tecnología Industrial
 # License: GPL-3.0
 # Project: KiBot (formerly KiPlot)
 """
@@ -24,9 +24,13 @@ import re
 from shutil import copy2
 import sys
 import sysconfig
+from ..error import KiPlotConfigurationError
 from ..gs import GS
 from .. import log
-from ..misc import W_NOCONFIG, W_NOKIENV, W_NOLIBS, W_NODEFSYMLIB, MISSING_WKS, W_MAXDEPTH, W_3DRESVER
+from ..misc import (W_NOCONFIG, W_NOKIENV, W_NOLIBS, W_NODEFSYMLIB, MISSING_WKS, W_MAXDEPTH, W_3DRESVER, W_LIBTVERSION,
+                    W_LIBTUNK)
+from .sexpdata import load, SExpData
+from .sexp_helpers import _check_is_symbol_list, _check_integer, _check_relaxed
 
 # Check python version to determine which version of ConfirParser to import
 if sys.version_info.major >= 3:
@@ -37,8 +41,10 @@ else:  # pragma: no cover (Py2)
 
 logger = log.get_logger()
 SYM_LIB_TABLE = 'sym-lib-table'
+FP_LIB_TABLE = 'fp-lib-table'
 KICAD_COMMON = 'kicad_common'
 MAXDEPTH = 20
+SUP_VERSION = 7
 reported = set()
 
 
@@ -106,10 +112,6 @@ def expand_env(val, env, extra_env, used_extra=None):
 
 class LibAlias(object):
     """ An entry for the symbol libs table """
-    libs_re = re.compile(r'\(name\s+(\S+|"(?:[^"]|\\")+")\)\s*\(type\s+(\S+|"(?:[^"]|\\")+")\)'
-                         r'\s*\(uri\s+(\S+|"(?:[^"]|\\")+")\)\s*\(options\s+(\S+|"(?:[^"]|\\")+")\)'
-                         r'\s*\(descr\s+(\S+|"(?:[^"]|\\")+")\)')
-
     def __init__(self):
         super().__init__()
         self.name = None
@@ -119,17 +121,23 @@ class LibAlias(object):
         self.descr = None
 
     @staticmethod
-    def parse(options, cline, env, extra_env):
+    def parse(items, env, extra_env):
-        m = LibAlias.libs_re.match(options)
+        s = LibAlias()
-        if not m:
+        for i in items[1:]:
-            raise KiConfError('Malformed lib entry', SYM_LIB_TABLE, cline, options)
+            i_type = _check_is_symbol_list(i)
-        lib = LibAlias()
+            if i_type == 'name':
-        lib.name = un_quote(m.group(1))
+                s.name = _check_relaxed(i, 1, i_type)
-        lib.legacy = m.group(2) == 'Legacy'
+            elif i_type == 'type':
-        lib.uri = os.path.abspath(expand_env(un_quote(m.group(3)), env, extra_env))
+                s.type = _check_relaxed(i, 1, i_type)
-        lib.options = un_quote(m.group(4))
+            elif i_type == 'uri':
-        lib.descr = un_quote(m.group(5))
+                s.uri = os.path.abspath(expand_env(_check_relaxed(i, 1, i_type), env, extra_env))
-        return lib
+            elif i_type == 'options':
+                s.options = _check_relaxed(i, 1, i_type)
+            elif i_type == 'descr':
+                s.descr = _check_relaxed(i, 1, i_type)
+            else:
+                logger.warning(W_LIBTUNK+'Unknown lib table attribute `{}`'.format(i))
+        return s
 
     def __str__(self):
         if not self.name:
@@ -148,7 +156,8 @@ class KiConf(object):
     models_3d_dir = None
     party_3rd_dir = None
     kicad_env = {}
-    lib_aliases = {}
+    lib_aliases = None
+    fp_aliases = None
     aliases_3D = {}
 
     def __init__(self):
@@ -162,9 +171,13 @@ class KiConf(object):
         KiConf.dirname = os.path.dirname(fname)
         KiConf.kicad_env['KIPRJMOD'] = KiConf.dirname
         KiConf.load_kicad_common()
-        KiConf.load_all_lib_aliases()
         KiConf.load_3d_aliases()
         KiConf.loaded = True
+        # Loaded on demand, here to debug
+        # KiConf.get_sym_lib_aliases()
+        # logger.error(KiConf.lib_aliases)
+        # KiConf.get_fp_lib_aliases()
+        # logger.error(KiConf.fp_aliases)
 
     def find_kicad_common():
         """ Looks for kicad_common config file.
@@ -431,42 +444,41 @@ class KiConf(object):
                 os.environ[k] = v
                 logger.debug('Exporting {}="{}"'.format(k, v))
 
-    def load_lib_aliases(fname):
+    def load_lib_aliases(fname, lib_aliases):
         if not os.path.isfile(fname):
             return False
         logger.debug('Loading symbols lib table `{}`'.format(fname))
         version = 0
         with open(fname, 'rt') as f:
-            line = f.readline().strip()
+            error = None
-            if line != '(sym_lib_table':
+            try:
-                raise KiConfError('Symbol libs table missing signature', fname, 1, line)
+                table = load(f)[0]
-            line = f.readline()
+            except SExpData as e:
-            cline = 2
+                error = str(e)
-            lib_regex = re.compile(r'\(lib\s*(.*)\)')
+            if error:
-            ver_regex = re.compile(r'\(version\s*(.*)\)')
+                raise KiPlotConfigurationError('Error loading `{}`: {}'.format(fname, error))
-            while line and line[0] != ')':
+        if not isinstance(table, list) or (table[0].value() != 'sym_lib_table' and table[0].value() != 'fp_lib_table'):
-                line = line.strip()
+            raise KiPlotConfigurationError('Error loading `{}`: not a library table'.format(fname))
-                m = lib_regex.match(line)
+        for e in table[1:]:
-                if m:
+            e_type = _check_is_symbol_list(e)
-                    alias = LibAlias.parse(m.group(1), cline, KiConf.kicad_env, {})
+            if e_type == 'version':
-                    if GS.debug_level > 1:
+                version = _check_integer(e, 1, e_type)
-                        logger.debug('- Adding lib alias '+str(alias))
+                if version > SUP_VERSION:
-                    KiConf.lib_aliases[alias.name] = alias
+                    logger.warning(W_LIBTVERSION+"Unsupported lib table version, loading could fail")
-                else:
+            elif e_type == 'lib':
-                    m = ver_regex.match(line)
+                alias = LibAlias.parse(e, KiConf.kicad_env, {})
-                    if m:
+                if GS.debug_level > 1:
-                        version = int(m.group(1))
+                    logger.debug('- Adding lib alias '+str(alias))
-                        logger.debug('Symbols library table version {}'.format(version))
+                lib_aliases[alias.name] = alias
-                    else:
+            else:
-                        raise KiConfError('Unknown symbol table entry', fname, cline, line)
+                logger.warning(W_LIBTUNK+"Unknown lib table entry `{}`".format(e_type))
-                line = f.readline()
-                cline += 1
         return True
 
-    def load_all_lib_aliases():
+    def load_all_lib_aliases(table_name, sys_dir, pattern):
         # Load the default symbol libs table.
         # This is the list of libraries enabled by the user.
         loaded = False
+        lib_aliases = {}
         if KiConf.config_dir:
             conf_dir = KiConf.config_dir
             if 'KICAD_CONFIG_HOME' in KiConf.kicad_env:
@@ -474,22 +486,39 @@ class KiConf(object):
                 # https://forum.kicad.info/t/kicad-config-home-inconsistencies-and-detail/26875
                 conf_dir = KiConf.kicad_env['KICAD_CONFIG_HOME']
                 logger.debug('Redirecting symbols lib table to '+conf_dir)
-            loaded = KiConf.load_lib_aliases(os.path.join(conf_dir, SYM_LIB_TABLE))
+            loaded = KiConf.load_lib_aliases(os.path.join(conf_dir, table_name), lib_aliases)
         if not loaded and 'KICAD_TEMPLATE_DIR' in KiConf.kicad_env:
-            loaded = KiConf.load_lib_aliases(os.path.join(KiConf.kicad_env['KICAD_TEMPLATE_DIR'], SYM_LIB_TABLE))
+            loaded = KiConf.load_lib_aliases(os.path.join(KiConf.kicad_env['KICAD_TEMPLATE_DIR'], table_name), lib_aliases)
         if not loaded:
             logger.warning(W_NODEFSYMLIB + 'Missing default symbol library table')
             # No default symbol libs table, try to create one
             if KiConf.sym_lib_dir:
-                for f in glob(os.path.join(KiConf.sym_lib_dir, '*.lib')):
+                logger.error(os.path.join(sys_dir, pattern))
+                for f in glob(os.path.join(sys_dir, pattern)):
                     alias = LibAlias()
                     alias.name = os.path.splitext(os.path.basename(f))[0]
                     alias.uri = f
                     if GS.debug_level > 1:
                         logger.debug('Detected lib alias '+str(alias))
-                    KiConf.lib_aliases[alias.name] = alias
+                    lib_aliases[alias.name] = alias
         # Load the project's table
-        KiConf.load_lib_aliases(os.path.join(KiConf.dirname, SYM_LIB_TABLE))
+        KiConf.load_lib_aliases(os.path.join(KiConf.dirname, table_name), lib_aliases)
+        return lib_aliases
+
+    def get_sym_lib_aliases(fname=None):
+        if KiConf.lib_aliases is None:
+            fname |= GS.sch_file
+            KiConf.init(fname)
+            pattern = '*.kicad_sym' if GS.ki6 else '*.lib'
+            KiConf.lib_aliases = KiConf.load_all_lib_aliases(SYM_LIB_TABLE, KiConf.sym_lib_dir, pattern)
+        return KiConf.lib_aliases
+
+    def get_fp_lib_aliases(fname=None):
+        if KiConf.fp_aliases is None:
+            fname |= GS.pcb_file
+            KiConf.init(fname)
+            KiConf.fp_aliases = KiConf.load_all_lib_aliases(FP_LIB_TABLE, KiConf.footprint_dir, '*.pretty')
+        return KiConf.fp_aliases
 
     def load_3d_aliases():
         if not KiConf.config_dir:

kibot/kicad/v5_sch.py

@@ -17,6 +17,7 @@ from datetime import datetime
 from copy import deepcopy
 from collections import OrderedDict
 from .config import KiConf, un_quote
+from .error import SchError, SchFileError, SchLibError
 from ..gs import GS
 from ..misc import (W_BADPOLI, W_POLICOORDS, W_BADSQUARE, W_BADCIRCLE, W_BADARC, W_BADTEXT, W_BADPIN, W_BADCOMP, W_BADDRAW,
                     W_UNKDCM, W_UNKAR, W_ARNOPATH, W_ARNOREF, W_MISCFLD, W_EXTRASPC, W_NOLIB, W_INCPOS, W_NOANNO, W_MISSLIB,
@@ -26,24 +27,6 @@ from .. import log
 logger = log.get_logger()
 
 
-class SchError(Exception):
-    pass
-
-
-class SchFileError(SchError):
-    def __init__(self, msg, code, reader):
-        super().__init__()
-        self.line = reader.line
-        self.file = reader.file
-        self.msg = msg
-        self.code = code
-
-
-class SchLibError(SchFileError):
-    def __init__(self, msg, code, reader):
-        super().__init__(msg, code, reader)
-
-
 class LineReader(object):
     def __init__(self, f, file):
         super().__init__()
@@ -1695,10 +1678,10 @@ class Schematic(object):
                         logger.debug('Filling desc for {}:{} `{}`'.format(c.lib, c.name, c.desc))
 
     def load_libs(self, fname):
-        KiConf.init(fname)
+        aliases = KiConf.get_sym_lib_aliases(fname)
         # Try to find the library paths
         for k in self.libs.keys():
-            alias = KiConf.lib_aliases.get(k)
+            alias = aliases.get(k)
             if k and alias:
                 self.libs[k] = alias.uri
                 if GS.debug_level > 1:

kibot/kicad/v6_sch.py

@@ -16,8 +16,12 @@ from collections import OrderedDict
 from ..gs import GS
 from .. import log
 from ..misc import W_NOLIB, W_UNKFLD, W_MISSCMP
-from .v5_sch import SchError, SchematicComponent, Schematic
+from .error import SchError
 from .sexpdata import load, SExpData, Symbol, dumps, Sep
+from .sexp_helpers import (_check_is_symbol_list, _check_len, _check_len_total, _check_symbol, _check_hide, _check_integer,
+                           _check_float, _check_str, _check_symbol_value, _check_symbol_float, _check_symbol_int,
+                           _check_symbol_str, _get_offset, _get_yes_no, _get_at, _get_size, _get_xy, _get_points)
+from .v5_sch import SchematicComponent, Schematic
 
 logger = log.get_logger()
 CROSSED_LIB = 'kibot_crossed'
@@ -28,143 +32,6 @@ SHEET_FILE = {'Sheet file', 'Sheetfile'}
 SHEET_NAME = {'Sheet name', 'Sheetname'}
 
 
-def _check_is_symbol_list(e, allow_orphan_symbol=()):
-    # Each entry is a list
-    if not isinstance(e, list):
-        if isinstance(e, Symbol):
-            name = e.value()
-            if name in allow_orphan_symbol:
-                return name
-            raise SchError('Orphan symbol `{}`'.format(e.value()))
-        else:
-            raise SchError('Orphan data `{}`'.format(e))
-    # The first element is a symbol
-    if not isinstance(e[0], Symbol):
-        raise SchError('Orphan data `{}`'.format(e[0]))
-    return e[0].value()
-
-
-def _check_len(items, pos, name):
-    if len(items) < pos+1:
-        raise SchError('Missing argument {} in `{}`'.format(pos, name))
-    return items[pos]
-
-
-def _check_len_total(items, num, name):
-    if len(items) != num:
-        raise SchError('Wrong number of attributes for {} `{}`'.format(name, items))
-
-
-def _check_symbol(items, pos, name):
-    value = _check_len(items, pos, name)
-    if not isinstance(value, Symbol):
-        raise SchError('{} is not a Symbol `{}`'.format(name, value))
-    return value.value()
-
-
-def _check_hide(items, pos, name):
-    value = _check_symbol(items, pos, name + ' hide')
-    if value != 'hide':
-        raise SchError('Found Symbol `{}` when `hide` expected'.format(value))
-    return True
-
-
-def _check_integer(items, pos, name):
-    value = _check_len(items, pos, name)
-    if not isinstance(value, int):
-        raise SchError('{} is not an integer `{}`'.format(name, value))
-    return value
-
-
-def _check_float(items, pos, name):
-    value = _check_len(items, pos, name)
-    if not isinstance(value, (float, int)):
-        raise SchError('{} is not a float `{}`'.format(name, value))
-    return value
-
-
-def _check_str(items, pos, name):
-    value = _check_len(items, pos, name)
-    if not isinstance(value, str):
-        raise SchError('{} is not a string `{}`'.format(name, value))
-    return value
-
-
-def _check_relaxed(items, pos, name):
-    value = _check_len(items, pos, name)
-    if isinstance(value, str):
-        return value
-    if isinstance(value, Symbol):
-        return value.value()
-    if isinstance(value, (float, int)):
-        return str(value)
-    raise SchError('{} is not a string, Symbol or number `{}`'.format(name, value))
-
-
-def _check_symbol_value(items, pos, name, sym):
-    value = _check_len(items, pos, name)
-    if not isinstance(value, list) or not isinstance(value[0], Symbol) or value[0].value() != sym:
-        raise SchError('Missing `{}` in `{}`'.format(sym, name))
-    return value
-
-
-def _check_symbol_float(items, pos, name, sym):
-    name += ' ' + sym
-    values = _check_symbol_value(items, pos, name, sym)
-    return _check_float(values, 1, name)
-
-
-def _check_symbol_int(items, pos, name, sym):
-    name += ' ' + sym
-    values = _check_symbol_value(items, pos, name, sym)
-    return _check_integer(values, 1, name)
-
-
-def _check_symbol_str(items, pos, name, sym):
-    name += ' ' + sym
-    values = _check_symbol_value(items, pos, name, sym)
-    return _check_str(values, 1, name)
-
-
-def _get_offset(items, pos, name):
-    value = _check_symbol_value(items, pos, name, 'offset')
-    return _check_float(value, 1, 'offset')
-
-
-def _get_yes_no(items, pos, name):
-    sym = _check_symbol(items, pos, name)
-    return sym == 'yes'
-
-
-def _get_id(items, pos, name):
-    value = _check_symbol_value(items, pos, name, 'id')
-    return _check_integer(value, 1, 'id')
-
-
-def _get_at(items, pos, name):
-    value = _check_symbol_value(items, pos, name, 'at')
-    angle = 0
-    if len(value) > 3:
-        angle = _check_float(value, 3, 'at angle')
-    return _check_float(value, 1, 'at x'), _check_float(value, 2, 'at y'), angle
-
-
-def _get_size(items, pos, name):
-    value = _check_symbol_value(items, pos, name, 'size')
-    return _get_xy(value)
-
-
-class Point(object):
-    def __init__(self, items):
-        super().__init__()
-        self.x = _check_float(items, 1, 'x coord')
-        self.y = _check_float(items, 2, 'y coord')
-
-    @staticmethod
-    def parse(items):
-        return Point(items)
-
-
 class PointXY(object):
     def __init__(self, x, y):
         super().__init__()
@@ -210,23 +77,6 @@ class Box(object):
             self.y2 = max(self.y2, b.y2)
 
 
-def _get_xy(items):
-    if len(items) != 3:
-        raise SchError('Point definition with wrong args (`{}`)'.format(items))
-    return Point.parse(items)
-
-
-def _get_points(items):
-    points = []
-    for i in items[1:]:
-        i_type = _check_is_symbol_list(i)
-        if i_type == 'xy':
-            points.append(_get_xy(i))
-        else:
-            raise SchError('Unknown points attribute `{}`'.format(i))
-    return points
-
-
 class FontEffects(object):
     """ Class used to describe text attributes """
     def __init__(self):

kibot/misc.py

@@ -259,6 +259,8 @@ W_BADPCB3DSTK = '(W115) '
 W_EEDA3D = '(W116) '
 W_MICROVIAS = '(W117) '
 W_BLINDVIAS = '(W118) '
+W_LIBTVERSION = '(W119) '
+W_LIBTUNK = '(W120) '
 # Somehow arbitrary, the colors are real, but can be different
 PCB_MAT_COLORS = {'fr1': "937042", 'fr2': "949d70", 'fr3': "adacb4", 'fr4': "332B16", 'fr5': "6cc290"}
 PCB_FINISH_COLORS = {'hal': "8b898c", 'hasl': "8b898c", 'imag': "8b898c", 'enig': "cfb96e", 'enepig': "cfb96e",