diff --git a/CHANGELOG.md b/CHANGELOG.md index ca782bff..5b32efcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Basic support for regular list items (#480) - Position: - Experimental support for gerber position files (#500) +- Copy Files: + - Mode to export the whole project (SCH, PCB, symbols, footprints, 3D models + and project files) (#491) - Help for the error levels - Warnings: - Explain about wrong dir/output separation (#493) @@ -98,6 +101,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 versions (#496) - Problems when using NET_NAME(n) for a value (#511) - JLCPCB rotations for bottom components +- Copy Files: + - Warnings when using both, the STEP and WRL model, of the same component + - Fail to detect 3D models subdirs when running alone ## [1.6.3] - 2023-06-26 diff --git a/docs/samples/generic_plot.kibot.yaml b/docs/samples/generic_plot.kibot.yaml index 5585661f..578b2c80 100644 --- a/docs/samples/generic_plot.kibot.yaml +++ b/docs/samples/generic_plot.kibot.yaml @@ -697,24 +697,29 @@ outputs: files: # [string=''] Destination directory inside the output dir, empty means the same of the file # relative to the source directory. + # Note that when you specify a name here files are copied to this destination + # without creating subdirs. The `project` mode is an exception. # For the `3d_models` type you can use DIR+ to create subdirs under DIR - dest: '' - # [string='.*'] A regular expression that source files must match + # [string='.*'] A regular expression that source files must match. + # Not used for the `project` mode filter: '.*' # [boolean=false] Only usable for the `3d_models` mode. - # Save a PCB copy modified to use the copied 3D models + # Save a PCB copy modified to use the copied 3D models. + # You don't need to specify it for `project` mode save_pcb: false - # [string='*'] File names to add, wildcards allowed. Use ** for recursive match. - # By default this pattern is applied to the current working dir. - # See the `from_outdir` option + # [string='*'] For the `files` and `out_files` mode this is th file names to add, + # wildcards allowed. Use ** for recursive match. + # For the `output` mode this is the name of the output. + # For the `3d_models` is a pattern to match the name of the 3D models extracted from the PCB. + # Not used for the `project` mode source: '*' - # [string='files'] [files,out_files,output,3d_models] How to interpret `source`. - # `files`: is a pattern for files relative to the working directory. - # `out_files`: is a pattern for files relative to output dir specified - # with `-d` command line option. - # `output`: is the name of an `output`. - # `3d_models`: is a pattern to match the name of the 3D models extracted - # from the PCB. + # [string='files'] [files,out_files,output,3d_models,project] From where do we get the files to be copied. + # `files`: files relative to the current working directory. + # `out_files`: files relative to output dir specified with `-d` command line option. + # `output`: files generated by the output specified by `source`. + # `3d_models`: 3D models used in the project. + # `project`: schematic, PCB, footprints, symbols, 3D models and project files (KiCad 6+) source_type: 'files' # [boolean=true] Store the file pointed by symlinks, not the symlink follow_links: true diff --git a/docs/source/configuration/outputs/copy_files.rst b/docs/source/configuration/outputs/copy_files.rst index b2396b02..d8142860 100644 --- a/docs/source/configuration/outputs/copy_files.rst +++ b/docs/source/configuration/outputs/copy_files.rst @@ -33,22 +33,27 @@ Parameters: - Valid keys: - - **source** :index:`: ` [string='*'] File names to add, wildcards allowed. Use ** for recursive match. - By default this pattern is applied to the current working dir. - See the `from_outdir` option. - - **source_type** :index:`: ` [string='files'] [files,out_files,output,3d_models] How to interpret `source`. - `files`: is a pattern for files relative to the working directory. - `out_files`: is a pattern for files relative to output dir specified - with `-d` command line option. - `output`: is the name of an `output`. - `3d_models`: is a pattern to match the name of the 3D models extracted - from the PCB.. + - **source** :index:`: ` [string='*'] For the `files` and `out_files` mode this is th file names to add, + wildcards allowed. Use ** for recursive match. + For the `output` mode this is the name of the output. + For the `3d_models` is a pattern to match the name of the 3D models extracted from the PCB. + Not used for the `project` mode. + - **source_type** :index:`: ` [string='files'] [files,out_files,output,3d_models,project] From where do we get the files to be copied. + `files`: files relative to the current working directory. + `out_files`: files relative to output dir specified with `-d` command line option. + `output`: files generated by the output specified by `source`. + `3d_models`: 3D models used in the project. + `project`: schematic, PCB, footprints, symbols, 3D models and project files (KiCad 6+). - ``dest`` :index:`: ` [string=''] Destination directory inside the output dir, empty means the same of the file relative to the source directory. + Note that when you specify a name here files are copied to this destination + without creating subdirs. The `project` mode is an exception. For the `3d_models` type you can use DIR+ to create subdirs under DIR. - ``filter`` :index:`: ` [string='.*'] A regular expression that source files must match. + Not used for the `project` mode. - ``save_pcb`` :index:`: ` [boolean=false] Only usable for the `3d_models` mode. Save a PCB copy modified to use the copied 3D models. + You don't need to specify it for `project` mode. - **no_virtual** :index:`: ` [boolean=false] Used to exclude 3D models for components with 'virtual' attribute. - ``dnf_filter`` :index:`: ` [string|list(string)='_none'] Name of the filter to mark components as not fitted. diff --git a/kibot/gs.py b/kibot/gs.py index fa9219cd..937da0a4 100644 --- a/kibot/gs.py +++ b/kibot/gs.py @@ -465,13 +465,14 @@ class GS(object): GS.exit_with_error('No SCH file found (*.sch), use -e to specify one.', EXIT_BAD_ARGS) @staticmethod - def copy_project(new_pcb_name): + def copy_project(new_pcb_name, dry=False): pro_name = GS.pro_file if pro_name is None or not os.path.isfile(pro_name): return None pro_copy = new_pcb_name.replace('.kicad_pcb', GS.pro_ext) - logger.debug('Copying project `{}` to `{}`'.format(pro_name, pro_copy)) - copy2(pro_name, pro_copy) + if not dry: + logger.debug('Copying project `{}` to `{}`'.format(pro_name, pro_copy)) + copy2(pro_name, pro_copy) return pro_copy @staticmethod @@ -757,3 +758,16 @@ class GS(object): if GS.ki6: return shape.ShowShape() return shape.ShowShape(shape.GetShape()) + + @staticmethod + def create_fp_lib(lib_name): + """ Create a new footprints lib. You must provide a path. + Doesn't fail if the lib is there. + .pretty extension is forced. """ + if not lib_name.endswith('.pretty'): + lib_name += '.pretty' + # FootprintLibCreate(PATH) doesn't check if the lib is there and aborts (no Python exception) if the directory exists + # So you must check it first and hence the abstraction is lost. + # This is why we just use os.makedirs + os.makedirs(lib_name, exist_ok=True) + return lib_name diff --git a/kibot/kicad/config.py b/kibot/kicad/config.py index d07bd553..184f7db8 100644 --- a/kibot/kicad/config.py +++ b/kibot/kicad/config.py @@ -29,7 +29,7 @@ from ..gs import GS from .. import log from ..misc import (W_NOCONFIG, W_NOKIENV, W_NOLIBS, W_NODEFSYMLIB, MISSING_WKS, W_MAXDEPTH, W_3DRESVER, W_LIBTVERSION, W_LIBTUNK) -from .sexpdata import load, SExpData +from .sexpdata import load, SExpData, Symbol, dumps, Sep from .sexp_helpers import _check_is_symbol_list, _check_integer, _check_relaxed # Check python version to determine which version of ConfirParser to import @@ -124,6 +124,7 @@ class LibAlias(object): self.uri = None self.options = None self.descr = None + self.type = None @staticmethod def parse(items, env, extra_env): @@ -142,6 +143,7 @@ class LibAlias(object): s.descr = _check_relaxed(i, 1, i_type) else: logger.warning(W_LIBTUNK+'Unknown lib table attribute `{}`'.format(i)) + s.legacy = s.type is not None and s.type != 'Legacy' return s def __str__(self): @@ -545,6 +547,29 @@ class KiConf(object): KiConf.fp_aliases = KiConf.load_all_lib_aliases(FP_LIB_TABLE, KiConf.footprint_dir, '*.pretty') return KiConf.fp_aliases + def save_fp_lib_aliases(fname, aliases, is_fp=True): + logger.debug(f'Writing lib table `{fname}`') + table = [Symbol('fp_lib_table' if is_fp else 'sym_lib_table'), Sep()] + for name in sorted(aliases.keys(), key=str.casefold): + alias = aliases[name] + cnt = [[Symbol('name'), alias.name], + [Symbol('type'), alias.type], + [Symbol('uri'), alias.uri]] + if alias.options is not None: + cnt.append([Symbol('options'), alias.options]) + if alias.descr is not None: + cnt.append([Symbol('descr'), alias.descr]) + table.append([Symbol('lib')] + cnt) + table.append(Sep()) + with open(fname, 'wt') as f: + f.write(dumps(table)) + f.write('\n') + + def fp_nick_to_path(nick): + fp_aliases = KiConf.get_fp_lib_aliases() + alias = fp_aliases.get(str(nick)) # UTF8 -> str + return alias + def load_3d_aliases(): if not KiConf.config_dir: return diff --git a/kibot/kicad/v6_sch.py b/kibot/kicad/v6_sch.py index 54a219b4..f0dcc1b2 100644 --- a/kibot/kicad/v6_sch.py +++ b/kibot/kicad/v6_sch.py @@ -1896,6 +1896,26 @@ class SchematicV6(Schematic): data.extend([s.write(cross), Sep()]) return [Sep(), Sep(), _symbol('lib_symbols', data), Sep()] + def write_lib(self, path, name, comps, do_back_up=False): + """ Creates a lib path/name containing the specified comps """ + lib = [Symbol('kicad_symbol_lib')] + # TODO: How do I know which is the valid version? Doesn't match the schematic version! + lib.append(_symbol('version', [20220914])) + lib.append(_symbol('generator', [Symbol("KiBot")])) + lib.append(Sep()) + for s in comps: + lib.extend([self.lib_symbol_names[name+':'+s].write(False), Sep()]) + # Keep a back-up of existing files + fname = os.path.join(path, name+'.kicad_sym') + if do_back_up and os.path.isfile(fname): + bkp = fname+'-bak' + os.replace(fname, bkp) + dirname = os.path.dirname(fname) + os.makedirs(dirname, exist_ok=True) + with open(fname, 'wt') as f: + f.write(dumps(lib)) + f.write('\n') + def save(self, fname=None, dest_dir=None, base_sheet=None, saved=None, cross=False, exp_hierarchy=False): # Switch to the current version global version diff --git a/kibot/out_copy_files.py b/kibot/out_copy_files.py index b7e4df94..89f49dbe 100644 --- a/kibot/out_copy_files.py +++ b/kibot/out_copy_files.py @@ -3,6 +3,7 @@ # Copyright (c) 2022-2023 Instituto Nacional de Tecnología Industrial # License: GPL-3.0 # Project: KiBot (formerly KiPlot) +from copy import copy import fnmatch import glob import os @@ -12,8 +13,8 @@ from sys import exit from .error import KiPlotConfigurationError from .gs import GS from .kiplot import config_output, get_output_dir, run_output -from .kicad.config import KiConf -from .misc import WRONG_ARGUMENTS, INTERNAL_ERROR, W_COPYOVER +from .kicad.config import KiConf, LibAlias, FP_LIB_TABLE, SYM_LIB_TABLE +from .misc import WRONG_ARGUMENTS, INTERNAL_ERROR, W_COPYOVER, W_MISSLIB, W_MISSCMP from .optionable import Optionable from .out_base_3d import Base3DOptions from .registrable import RegOutput @@ -35,43 +36,46 @@ class FilesList(Optionable): super().__init__() with document: self.source = '*' - """ *File names to add, wildcards allowed. Use ** for recursive match. - By default this pattern is applied to the current working dir. - See the `from_outdir` option """ + """ *For the `files` and `out_files` mode this is th file names to add, + wildcards allowed. Use ** for recursive match. + For the `output` mode this is the name of the output. + For the `3d_models` is a pattern to match the name of the 3D models extracted from the PCB. + Not used for the `project` mode """ self.source_type = 'files' - """ *[files,out_files,output,3d_models] How to interpret `source`. - `files`: is a pattern for files relative to the working directory. - `out_files`: is a pattern for files relative to output dir specified - with `-d` command line option. - `output`: is the name of an `output`. - `3d_models`: is a pattern to match the name of the 3D models extracted - from the PCB. """ + """ *[files,out_files,output,3d_models,project] From where do we get the files to be copied. + `files`: files relative to the current working directory. + `out_files`: files relative to output dir specified with `-d` command line option. + `output`: files generated by the output specified by `source`. + `3d_models`: 3D models used in the project. + `project`: schematic, PCB, footprints, symbols, 3D models and project files (KiCad 6+) """ self.filter = '.*' - """ A regular expression that source files must match """ + """ A regular expression that source files must match. + Not used for the `project` mode """ self.dest = '' """ Destination directory inside the output dir, empty means the same of the file relative to the source directory. + Note that when you specify a name here files are copied to this destination + without creating subdirs. The `project` mode is an exception. For the `3d_models` type you can use DIR+ to create subdirs under DIR """ self.save_pcb = False """ Only usable for the `3d_models` mode. - Save a PCB copy modified to use the copied 3D models """ + Save a PCB copy modified to use the copied 3D models. + You don't need to specify it for `project` mode """ + self._append_mode = False def apply_rename(self, fname): - is_abs = os.path.isabs(fname) - append_mode = self.dest and self.dest[-1] == '+' - if self.dest and not append_mode: + if self.dest and not self._append_mode: # A destination specified by the user dest = os.path.basename(fname) - elif is_abs: + else: for d in self.rel_dirs: if d is not None and fname.startswith(d): dest = os.path.relpath(fname, d) break else: dest = os.path.basename(fname) - else: - dest = os.path.relpath(fname, os.getcwd()) - return '${KIPRJMOD}/'+os.path.join(self.output_dir, dest) + res = '${KIPRJMOD}/'+os.path.join(self.output_dir, dest) + return res class Copy_FilesOptions(Base3DOptions): @@ -118,24 +122,124 @@ class Copy_FilesOptions(Base3DOptions): exit(INTERNAL_ERROR) return files_list - def get_3d_models(self, f): + def copy_footprints(self, dest, dry): + out_lib_base = os.path.join(self.output_dir, dest, 'footprints') + out_lib_base_prj = os.path.join('${KIPRJMOD}', 'footprints') + aliases = {} + extra_files = [] + added = set() + for m in GS.get_modules(): + id = m.GetFPID() + lib_nick = str(id.GetLibNickname()) + src_alias = KiConf.fp_nick_to_path(lib_nick) + if src_alias is None: + logger.warning(f'{W_MISSLIB}Missing footprint library `{lib_nick}`') + continue + src_lib = src_alias.uri + out_lib = os.path.join(out_lib_base, lib_nick) + out_lib = GS.create_fp_lib(out_lib) + if lib_nick not in aliases: + new_alias = copy(src_alias) + new_alias.uri = os.path.join(out_lib_base_prj, lib_nick+'.pretty') + aliases[lib_nick] = new_alias + + name = str(id.GetLibItemName()) + mod_fname = name+'.kicad_mod' + footprint_src = os.path.join(src_lib, mod_fname) + if not os.path.isfile(footprint_src): + logger.warning(f'{W_MISSCMP}Missing footprint `{name}` ({lib_nick}:{name})') + elif footprint_src not in added: + footprint_dst = os.path.join(out_lib, mod_fname) + extra_files.append((footprint_src, footprint_dst)) + added.add(footprint_src) + table_fname = os.path.join(self.output_dir, dest, FP_LIB_TABLE) + extra_files.append(table_fname) + if not dry: + KiConf.save_fp_lib_aliases(table_fname, aliases) + return extra_files + + def copy_symbols(self, dest, dry): + extra_files = [] + if not GS.sch: + return extra_files + out_lib_base = os.path.join(self.output_dir, dest, 'symbols') + out_lib_base_prj = os.path.join('${KIPRJMOD}', 'symbols') + aliases = {} + # Split the collected components into separated libs + libs = {} + for obj in GS.sch.lib_symbol_names.values(): + lib = obj.lib if obj.lib else 'locally_edited' + libs.setdefault(lib, []).append(obj.name) + table_fname = os.path.join(self.output_dir, dest, SYM_LIB_TABLE) + extra_files.append(table_fname) + if dry: + for lib in libs.keys(): + extra_files.append(os.path.join(out_lib_base, lib+'.kicad_sym')) + else: + # Create the libs + for lib, comps in libs.items(): + GS.sch.write_lib(out_lib_base, lib, comps) + new_alias = LibAlias() + new_alias.name = lib + new_alias.legacy = False + new_alias.type = 'KiCad' + new_alias.options = new_alias.descr = '' + new_alias.uri = os.path.join(out_lib_base_prj, lib+'.kicad_sym') + aliases[lib] = new_alias + extra_files.append(os.path.join(out_lib_base, lib+'.kicad_sym')) + # Create the sym-lib-table + KiConf.save_fp_lib_aliases(table_fname, aliases, is_fp=False) + return extra_files + + def add_sch_files(self, files, dest_dir): + for f in GS.sch.get_files(): + files.append(os.path.join(dest_dir, os.path.relpath(f, GS.sch_dir))) + + def get_3d_models(self, f, mode_project, dry): """ Look for the 3D models and make a list, optionally download them """ + extra_files = [] GS.check_pcb() GS.load_board() - dest_dir = f.dest - if dest_dir and dest_dir[-1] == '+': - dest_dir = dest_dir[:-1] - f.output_dir = dest_dir + if mode_project: + # From the PCB point this is just the 3D models dir + f.output_dir = '3d_models' + f._append_mode = True + else: + dest_dir = f.dest + f._append_mode = False + if dest_dir and dest_dir[-1] == '+': + dest_dir = dest_dir[:-1] + f._append_mode = True + f.output_dir = dest_dir # Apply any variant self.filter_pcb_components(do_3D=True, do_2D=True) - # Download missing models and rename all collect 3D models (renamed) + # Download missing models and rename all collected 3D models (renamed) f.rel_dirs = self.rel_dirs files_list = self.download_models(rename_filter=f.source, rename_function=FilesList.apply_rename, rename_data=f) - if f.save_pcb: - fname = os.path.join(self.output_dir, os.path.basename(GS.pcb_file)) - logger.debug('Saving the PCB to '+fname) - GS.board.Save(fname) - GS.copy_project(fname) + + if f.save_pcb or mode_project: + dest_dir = self.output_dir + if mode_project: + dest_dir = os.path.join(dest_dir, f.dest) + os.makedirs(dest_dir, exist_ok=True) + fname = os.path.join(dest_dir, os.path.basename(GS.pcb_file)) + if not dry: + logger.debug('Saving the PCB to '+fname) + GS.board.Save(fname) + if mode_project: + logger.debug('Saving the schematic to '+dest_dir) + GS.sch.save_variant(dest_dir) + self.add_sch_files(extra_files, dest_dir) + elif mode_project: + self.add_sch_files(extra_files, dest_dir) + prj_name = GS.copy_project(fname, dry) + # Extra files that we are generating + extra_files.append(fname) + if prj_name: + extra_files.append(prj_name) + if mode_project: + extra_files += self.copy_footprints(f.dest, dry) + extra_files += self.copy_symbols(f.dest, dry) if not self._comps: # We must undo the download/rename self.undo_3d_models_rename(GS.board) @@ -152,60 +256,79 @@ class Copy_FilesOptions(Base3DOptions): fn = fn[:-5]+'.wrl' if os.path.isfile(fn) and fn not in files_list: new_list.append(fn) - return files_list+fnmatch.filter(new_list, f.source) + if mode_project: + # From the output point this needs to add the destination dir + f.output_dir = os.path.join(f.dest, f.output_dir) + return files_list+fnmatch.filter(new_list, f.source), extra_files def get_files(self, no_out_run=False): files = [] + # The source file can be relative to the current directory or to the output directory src_dir_cwd = os.getcwd() src_dir_outdir = self.expand_filename_sch(GS.out_dir) # Initialize the config class so we can know where are the 3D models at system level KiConf.init(GS.pcb_file) + # List of base paths self.rel_dirs = [] if KiConf.models_3d_dir: self.rel_dirs.append(os.path.normpath(os.path.join(GS.pcb_dir, KiConf.models_3d_dir))) if KiConf.party_3rd_dir: self.rel_dirs.append(os.path.normpath(os.path.join(GS.pcb_dir, KiConf.party_3rd_dir))) self.rel_dirs.append(GS.pcb_dir) + # Process each file specification expanding it to real files for f in self.files: - from_outdir = False - if f.source_type == 'out_files' or f.source_type == 'output': - from_outdir = True - src_dir = src_dir_outdir if from_outdir else src_dir_cwd + src_dir = src_dir_outdir if f.source_type == 'out_files' or f.source_type == 'output' else src_dir_cwd + mode_project = f.source_type == 'project' + if mode_project and not GS.ki6: + raise KiPlotConfigurationError('The `project` mode needs KiCad 6 or newer') mode_3d = f.source_type == '3d_models' - mode_3d_append = mode_3d and f.dest and f.dest[-1] == '+' # Get the list of candidates files_list = None + extra_files = [] if f.source_type == 'output': + # The files are generated by a KiBot output files_list = self.get_from_output(f, no_out_run) - elif mode_3d: - files_list = self.get_3d_models(f) + elif mode_3d or mode_project: + # The files are 3D models + files_list, extra_files = self.get_3d_models(f, mode_project, no_out_run) else: # files and out_files + # Regular files source = f.expand_filename_both(f.source, make_safe=False) files_list = glob.iglob(os.path.join(src_dir, source), recursive=True) if GS.debug_level > 1: files_list = list(files_list) logger.debug('- Pattern {} list of files: {}'.format(source, files_list)) # Filter and adapt them - for fname in filter(re.compile(f.filter).match, files_list): + fil = re.compile(f.filter) + for fname in filter(fil.match, files_list): fname_real = os.path.realpath(fname) if self.follow_links else os.path.abspath(fname) # Compute the destination directory dest = fname is_abs = os.path.isabs(fname) - if f.dest and not mode_3d_append: + if f.dest and not f._append_mode: # A destination specified by the user + # All files goes to the same destination directory dest = os.path.join(f.dest, os.path.basename(fname)) - elif mode_3d and is_abs: + elif (mode_3d or mode_project) and is_abs: for d in self.rel_dirs: if d is not None and fname.startswith(d): dest = os.path.relpath(dest, d) break else: dest = os.path.basename(fname) - if mode_3d_append: - dest = os.path.join(f.dest[:-1], dest) + if f._append_mode: + dest = os.path.join(f.output_dir, dest) else: dest = os.path.relpath(dest, src_dir) files.append((fname_real, dest)) + # Process the special extra files + for f in extra_files: + if isinstance(f, str): + if fil.match(f): + files.append((None, f)) + else: + if fil.match(f[0]): + files.append(f) return files def get_targets(self, out_dir): @@ -215,7 +338,7 @@ class Copy_FilesOptions(Base3DOptions): def get_dependencies(self): files = self.get_files(no_out_run=True) - return sorted([v for v, _ in files]) + return sorted([v for v, _ in files if v is not None]) def run(self, output): super().run(output) @@ -227,6 +350,9 @@ class Copy_FilesOptions(Base3DOptions): output += os.path.sep copied = {} for (src, dst) in files: + if src is None: + # Files we generate, we don't need to copy them + continue dest = os.path.join(output, dst) dest_dir = os.path.dirname(dest) if not os.path.isdir(dest_dir):