415 lines
18 KiB
Python
415 lines
18 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2022-2023 Salvador E. Tropea
|
|
# Copyright (c) 2022-2023 Instituto Nacional de Tecnología Industrial
|
|
# License: GPL-3.0
|
|
# Project: KiBot (formerly KiPlot)
|
|
from copy import copy
|
|
import fnmatch
|
|
import glob
|
|
import os
|
|
import re
|
|
from shutil import copy2
|
|
from sys import exit
|
|
from .error import KiPlotConfigurationError
|
|
from .gs import GS
|
|
from .kiplot import config_output, get_output_dir, run_output
|
|
from .kicad.config import KiConf, LibAlias, FP_LIB_TABLE, SYM_LIB_TABLE
|
|
from .misc import WRONG_ARGUMENTS, INTERNAL_ERROR, W_COPYOVER, W_MISSLIB, W_MISSCMP
|
|
from .optionable import Optionable
|
|
from .out_base_3d import Base3DOptions
|
|
from .registrable import RegOutput
|
|
from .macros import macros, document, output_class # noqa: F401
|
|
from . import log
|
|
|
|
logger = log.get_logger()
|
|
|
|
|
|
def may_be_rel(file):
|
|
rel_file = os.path.relpath(file)
|
|
if len(rel_file) < len(file):
|
|
return rel_file
|
|
return file
|
|
|
|
|
|
class FilesList(Optionable):
|
|
def __init__(self):
|
|
super().__init__()
|
|
with document:
|
|
self.source = '*'
|
|
""" *For the `files` and `out_files` mode this is th file names to add,
|
|
wildcards allowed. Use ** for recursive match.
|
|
For the `output` mode this is the name of the output.
|
|
For the `3d_models` is a pattern to match the name of the 3D models extracted from the PCB.
|
|
Not used for the `project` mode """
|
|
self.source_type = 'files'
|
|
""" *[files,out_files,output,3d_models,project] From where do we get the files to be copied.
|
|
`files`: files relative to the current working directory.
|
|
`out_files`: files relative to output dir specified with `-d` command line option.
|
|
`output`: files generated by the output specified by `source`.
|
|
`3d_models`: 3D models used in the project.
|
|
`project`: schematic, PCB, footprints, symbols, 3D models and project files (KiCad 6+) """
|
|
self.filter = '.*'
|
|
""" A regular expression that source files must match.
|
|
Not used for the `project` mode """
|
|
self.dest = ''
|
|
""" Destination directory inside the output dir, empty means the same of the file
|
|
relative to the source directory.
|
|
Note that when you specify a name here files are copied to this destination
|
|
without creating subdirs. The `project` mode is an exception.
|
|
For the `3d_models` type you can use DIR+ to create subdirs under DIR """
|
|
self.save_pcb = False
|
|
""" Only usable for the `3d_models` mode.
|
|
Save a PCB copy modified to use the copied 3D models.
|
|
You don't need to specify it for `project` mode """
|
|
self._append_mode = False
|
|
|
|
def apply_rename(self, fname):
|
|
if self.dest and not self._append_mode:
|
|
# A destination specified by the user
|
|
dest = os.path.basename(fname)
|
|
else:
|
|
for d in self.rel_dirs:
|
|
if d is not None and fname.startswith(d):
|
|
dest = os.path.relpath(fname, d)
|
|
break
|
|
else:
|
|
dest = os.path.basename(fname)
|
|
res = '${KIPRJMOD}/'+os.path.join(self.output_dir, dest)
|
|
return res
|
|
|
|
|
|
class Copy_FilesOptions(Base3DOptions):
|
|
def __init__(self):
|
|
with document:
|
|
self.files = FilesList
|
|
""" *[list(dict)] Which files will be included """
|
|
self.follow_links = True
|
|
""" Store the file pointed by symlinks, not the symlink """
|
|
self.link_no_copy = False
|
|
""" Create symlinks instead of copying files """
|
|
super().__init__()
|
|
self._expand_id = 'copy'
|
|
self._expand_ext = 'files'
|
|
|
|
def config(self, parent):
|
|
super().config(parent)
|
|
if isinstance(self.files, type):
|
|
raise KiPlotConfigurationError('No files provided')
|
|
|
|
def get_from_output(self, f, no_out_run):
|
|
from_output = f.source
|
|
logger.debugl(2, '- From output `{}`'.format(from_output))
|
|
out = RegOutput.get_output(from_output)
|
|
if out is not None:
|
|
config_output(out)
|
|
out_dir = get_output_dir(out.dir, out, dry=True)
|
|
files_list = out.get_targets(out_dir)
|
|
logger.debugl(2, '- List of files: {}'.format(files_list))
|
|
else:
|
|
logger.error('Unknown output `{}` selected in {}'.format(from_output, self._parent))
|
|
exit(WRONG_ARGUMENTS)
|
|
# Check if we must run the output to create the files
|
|
if not no_out_run:
|
|
for file in files_list:
|
|
if not os.path.isfile(file):
|
|
# The target doesn't exist
|
|
if not out._done:
|
|
# The output wasn't created in this run, try running it
|
|
run_output(out)
|
|
if not os.path.isfile(file):
|
|
# Still missing, something is wrong
|
|
logger.error('Unable to generate `{}` from {}'.format(file, out))
|
|
exit(INTERNAL_ERROR)
|
|
return files_list
|
|
|
|
def copy_footprints(self, dest, dry):
|
|
out_lib_base = os.path.join(self.output_dir, dest, 'footprints')
|
|
out_lib_base_prj = os.path.join('${KIPRJMOD}', 'footprints')
|
|
aliases = {}
|
|
extra_files = []
|
|
added = set()
|
|
for m in GS.get_modules():
|
|
id = m.GetFPID()
|
|
lib_nick = str(id.GetLibNickname())
|
|
src_alias = KiConf.fp_nick_to_path(lib_nick)
|
|
if src_alias is None:
|
|
logger.warning(f'{W_MISSLIB}Missing footprint library `{lib_nick}`')
|
|
continue
|
|
src_lib = src_alias.uri
|
|
out_lib = os.path.join(out_lib_base, lib_nick)
|
|
out_lib = GS.create_fp_lib(out_lib)
|
|
if lib_nick not in aliases:
|
|
new_alias = copy(src_alias)
|
|
new_alias.uri = os.path.join(out_lib_base_prj, lib_nick+'.pretty')
|
|
aliases[lib_nick] = new_alias
|
|
|
|
name = str(id.GetLibItemName())
|
|
mod_fname = name+'.kicad_mod'
|
|
footprint_src = os.path.join(src_lib, mod_fname)
|
|
if not os.path.isfile(footprint_src):
|
|
logger.warning(f'{W_MISSCMP}Missing footprint `{name}` ({lib_nick}:{name})')
|
|
elif footprint_src not in added:
|
|
footprint_dst = os.path.join(out_lib, mod_fname)
|
|
extra_files.append((footprint_src, footprint_dst))
|
|
added.add(footprint_src)
|
|
table_fname = os.path.join(self.output_dir, dest, FP_LIB_TABLE)
|
|
extra_files.append(table_fname)
|
|
if not dry:
|
|
KiConf.save_fp_lib_aliases(table_fname, aliases)
|
|
return extra_files
|
|
|
|
def copy_symbols(self, dest, dry):
|
|
extra_files = []
|
|
if not GS.sch:
|
|
return extra_files
|
|
out_lib_base = os.path.join(self.output_dir, dest, 'symbols')
|
|
out_lib_base_prj = os.path.join('${KIPRJMOD}', 'symbols')
|
|
aliases = {}
|
|
# Split the collected components into separated libs
|
|
libs = {}
|
|
for obj in GS.sch.lib_symbol_names.values():
|
|
lib = obj.lib if obj.lib else 'locally_edited'
|
|
libs.setdefault(lib, []).append(obj.name)
|
|
table_fname = os.path.join(self.output_dir, dest, SYM_LIB_TABLE)
|
|
extra_files.append(table_fname)
|
|
if dry:
|
|
for lib in libs.keys():
|
|
if lib != 'locally_edited':
|
|
extra_files.append(os.path.join(out_lib_base, lib+'.kicad_sym'))
|
|
else:
|
|
# Create the libs
|
|
for lib, comps in libs.items():
|
|
if lib == 'locally_edited':
|
|
# Not from a lib, just a copy inside the SCH
|
|
continue
|
|
GS.sch.write_lib(out_lib_base, lib, comps)
|
|
new_alias = LibAlias()
|
|
new_alias.name = lib
|
|
new_alias.legacy = False
|
|
new_alias.type = 'KiCad'
|
|
new_alias.options = new_alias.descr = ''
|
|
new_alias.uri = os.path.join(out_lib_base_prj, lib+'.kicad_sym')
|
|
aliases[lib] = new_alias
|
|
extra_files.append(os.path.join(out_lib_base, lib+'.kicad_sym'))
|
|
# Create the sym-lib-table
|
|
KiConf.save_fp_lib_aliases(table_fname, aliases, is_fp=False)
|
|
return extra_files
|
|
|
|
def add_sch_files(self, files, dest_dir):
|
|
for f in GS.sch.get_files():
|
|
files.append(os.path.join(dest_dir, os.path.relpath(f, GS.sch_dir)))
|
|
|
|
def get_3d_models(self, f, mode_project, dry):
|
|
""" Look for the 3D models and make a list, optionally download them """
|
|
extra_files = []
|
|
GS.check_pcb()
|
|
GS.load_board()
|
|
if mode_project:
|
|
# From the PCB point this is just the 3D models dir
|
|
f.output_dir = '3d_models'
|
|
f._append_mode = True
|
|
else:
|
|
dest_dir = f.dest
|
|
f._append_mode = False
|
|
if dest_dir and dest_dir[-1] == '+':
|
|
dest_dir = dest_dir[:-1]
|
|
f._append_mode = True
|
|
f.output_dir = dest_dir
|
|
# Apply any variant
|
|
self.filter_pcb_components(do_3D=True, do_2D=True)
|
|
# Download missing models and rename all collected 3D models (renamed)
|
|
f.rel_dirs = self.rel_dirs
|
|
files_list = self.download_models(rename_filter=f.source, rename_function=FilesList.apply_rename, rename_data=f)
|
|
|
|
if f.save_pcb or mode_project:
|
|
dest_dir = self.output_dir
|
|
if mode_project:
|
|
dest_dir = os.path.join(dest_dir, f.dest)
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
fname = os.path.join(dest_dir, os.path.basename(GS.pcb_file))
|
|
if not dry:
|
|
logger.debug('Saving the PCB to '+fname)
|
|
GS.board.Save(fname)
|
|
if mode_project:
|
|
logger.debug('Saving the schematic to '+dest_dir)
|
|
GS.sch.save_variant(dest_dir)
|
|
self.add_sch_files(extra_files, dest_dir)
|
|
elif mode_project:
|
|
self.add_sch_files(extra_files, dest_dir)
|
|
prj_name, prl_name, dru_name = GS.copy_project(fname, dry)
|
|
# Extra files that we are generating
|
|
extra_files.append(fname)
|
|
if prj_name:
|
|
extra_files.append(prj_name)
|
|
if prl_name:
|
|
extra_files.append(prl_name)
|
|
if dru_name:
|
|
extra_files.append(dru_name)
|
|
# Worksheet
|
|
wks = GS.fix_page_layout(prj_name, dry=dry)
|
|
extra_files += [w for w in wks if w is not None]
|
|
if mode_project:
|
|
extra_files += self.copy_footprints(f.dest, dry)
|
|
extra_files += self.copy_symbols(f.dest, dry)
|
|
if not self._comps:
|
|
# We must undo the download/rename
|
|
self.undo_3d_models_rename(GS.board)
|
|
else:
|
|
self.unfilter_pcb_components(do_3D=True, do_2D=True)
|
|
# Also include the step/wrl counterpart
|
|
new_list = []
|
|
for fn in files_list:
|
|
if fn.endswith('.wrl'):
|
|
fn = fn[:-4]+'.step'
|
|
if os.path.isfile(fn) and fn not in files_list:
|
|
new_list.append(fn)
|
|
elif fn.endswith('.step'):
|
|
fn = fn[:-5]+'.wrl'
|
|
if os.path.isfile(fn) and fn not in files_list:
|
|
new_list.append(fn)
|
|
if mode_project:
|
|
# From the output point this needs to add the destination dir
|
|
f.output_dir = os.path.join(f.dest, f.output_dir)
|
|
return files_list+fnmatch.filter(new_list, f.source), extra_files
|
|
|
|
def get_files(self, no_out_run=False):
|
|
files = []
|
|
# The source file can be relative to the current directory or to the output directory
|
|
src_dir_cwd = os.getcwd()
|
|
src_dir_outdir = self.expand_filename_sch(GS.out_dir)
|
|
# Initialize the config class so we can know where are the 3D models at system level
|
|
KiConf.init(GS.pcb_file)
|
|
# List of base paths
|
|
self.rel_dirs = []
|
|
if KiConf.models_3d_dir:
|
|
self.rel_dirs.append(os.path.normpath(os.path.join(GS.pcb_dir, KiConf.models_3d_dir)))
|
|
if KiConf.party_3rd_dir:
|
|
self.rel_dirs.append(os.path.normpath(os.path.join(GS.pcb_dir, KiConf.party_3rd_dir)))
|
|
self.rel_dirs.append(GS.pcb_dir)
|
|
# Process each file specification expanding it to real files
|
|
for f in self.files:
|
|
src_dir = src_dir_outdir if f.source_type == 'out_files' or f.source_type == 'output' else src_dir_cwd
|
|
mode_project = f.source_type == 'project'
|
|
if mode_project and not GS.ki6:
|
|
raise KiPlotConfigurationError('The `project` mode needs KiCad 6 or newer')
|
|
mode_3d = f.source_type == '3d_models'
|
|
# Get the list of candidates
|
|
files_list = None
|
|
extra_files = []
|
|
if f.source_type == 'output':
|
|
# The files are generated by a KiBot output
|
|
files_list = self.get_from_output(f, no_out_run)
|
|
elif mode_3d or mode_project:
|
|
# The files are 3D models
|
|
files_list, extra_files = self.get_3d_models(f, mode_project, no_out_run)
|
|
else: # files and out_files
|
|
# Regular files
|
|
source = f.expand_filename_both(f.source, make_safe=False)
|
|
files_list = glob.iglob(os.path.join(src_dir, source), recursive=True)
|
|
if GS.debug_level > 1:
|
|
files_list = list(files_list)
|
|
logger.debug('- Pattern {} list of files: {}'.format(source, files_list))
|
|
# Filter and adapt them
|
|
fil = re.compile(f.filter)
|
|
for fname in filter(fil.match, files_list):
|
|
fname_real = os.path.realpath(fname) if self.follow_links else os.path.abspath(fname)
|
|
# Compute the destination directory
|
|
dest = fname
|
|
is_abs = os.path.isabs(fname)
|
|
if f.dest and not f._append_mode:
|
|
# A destination specified by the user
|
|
# All files goes to the same destination directory
|
|
dest = os.path.join(f.dest, os.path.basename(fname))
|
|
elif (mode_3d or mode_project) and is_abs:
|
|
for d in self.rel_dirs:
|
|
if d is not None and fname.startswith(d):
|
|
dest = os.path.relpath(dest, d)
|
|
break
|
|
else:
|
|
dest = os.path.basename(fname)
|
|
if f._append_mode:
|
|
dest = os.path.join(f.output_dir, dest)
|
|
else:
|
|
dest = os.path.relpath(dest, src_dir)
|
|
files.append((fname_real, dest))
|
|
# Process the special extra files
|
|
for f in extra_files:
|
|
if isinstance(f, str):
|
|
if fil.match(f):
|
|
files.append((None, f))
|
|
else:
|
|
if fil.match(f[0]):
|
|
files.append(f)
|
|
return files
|
|
|
|
def get_targets(self, out_dir):
|
|
self.output_dir = out_dir
|
|
files = self.get_files(no_out_run=True)
|
|
return sorted([os.path.join(out_dir, v) for _, v in files])
|
|
|
|
def get_dependencies(self):
|
|
files = self.get_files(no_out_run=True)
|
|
return sorted([v for v, _ in files if v is not None])
|
|
|
|
def run(self, output):
|
|
super().run(output)
|
|
# Output file name
|
|
logger.debug('Collecting files')
|
|
# Collect the files
|
|
files = self.get_files()
|
|
logger.debug('Copying files')
|
|
output += os.path.sep
|
|
copied = {}
|
|
for (src, dst) in files:
|
|
if src is None:
|
|
# Files we generate, we don't need to copy them
|
|
continue
|
|
dest = os.path.join(output, dst)
|
|
dest_dir = os.path.dirname(dest)
|
|
if not os.path.isdir(dest_dir):
|
|
os.makedirs(dest_dir)
|
|
logger.debug('- {} -> {}'.format(src, dest))
|
|
if dest in copied:
|
|
logger.warning(W_COPYOVER+'`{}` and `{}` both are copied to `{}`'.
|
|
format(may_be_rel(src), may_be_rel(copied[dest]), may_be_rel(dest)))
|
|
try:
|
|
if os.path.samefile(src, dest):
|
|
raise KiPlotConfigurationError('Trying to copy {} over itself {}'.format(src, dest))
|
|
except FileNotFoundError:
|
|
pass
|
|
if os.path.isfile(dest) or os.path.islink(dest):
|
|
os.remove(dest)
|
|
if self.link_no_copy:
|
|
os.symlink(os.path.relpath(src, os.path.dirname(dest)), dest)
|
|
else:
|
|
copy2(src, dest)
|
|
copied[dest] = src
|
|
# Remove the downloaded 3D models
|
|
self.remove_temporals()
|
|
|
|
|
|
@output_class
|
|
class Copy_Files(BaseOutput): # noqa: F821
|
|
""" Files copier
|
|
Used to copy files to the output directory.
|
|
Useful when an external tool is used to compress the output directory.
|
|
Note that you can use the `compress` output to create archives """
|
|
def __init__(self):
|
|
super().__init__()
|
|
# Make it low priority so it gets created after all the other outputs
|
|
self.priority = 11
|
|
with document:
|
|
self.options = Copy_FilesOptions
|
|
""" *[dict] Options for the `copy_files` output """
|
|
self._none_related = True
|
|
# The help is inherited and already mentions the default priority
|
|
self.fix_priority_help()
|
|
|
|
def get_dependencies(self):
|
|
return self.options.get_dependencies()
|
|
|
|
def run(self, output_dir):
|
|
# No output member, just a dir
|
|
self.options.output_dir = output_dir
|
|
self.options.run(output_dir)
|