From 38ea584fb11c7e673bd4407aafcc863e436b3d9c Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 18 Aug 2023 00:10:07 -0500 Subject: [PATCH 01/27] Add initial script --- config.json | 136 ++++++++++++++++++++++++++++++++++++++++++++++ create_libpack.py | 90 ++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 config.json create mode 100644 create_libpack.py diff --git a/config.json b/config.json new file mode 100644 index 0000000..ddaf4be --- /dev/null +++ b/config.json @@ -0,0 +1,136 @@ +{ + "FreeCAD-version":"0.22", + "LibPack-version":"3.0.0", + "contents": [ + { + "name":"python", + "git-repo":"https://github.com/python/cpython", + "git-ref":"v3.11.4" + }, + { + "name":"qt", + "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" + }, + { + "name":"boost", + "git-repo":"https://github.com/boostorg/boost", + "git-ref":"boost-1.83.0" + }, + { + "name":"coin", + "git-repo":"https://github.com/coin3d/coin", + "git-ref":"master" + }, + { + "name":"quarter", + "git-repo":"https://github.com/coin3d/quarter", + "git-ref":"master" + }, + { + "name":"swig", + "url":"http://prdownloads.sourceforge.net/swig/swigwin-4.1.1.zip" + }, + { + "name":"pivy", + "git-repo":"https://github.com/coin3d/pivy", + "git-ref":"master" + }, + { + "name":"libclang", + "url":"https://download.qt.io/development_releases/prebuilt/libclang/libclang-release_140-based-windows-vs2019_64.7z" + }, + { + "name":"pyside", + "pip-install":"pyside6==6.5.2" + }, + { + "name":"vtk", + "git-repo":"https://gitlab.kitware.com/vtk/vtk.git", + "git-ref":"v9.2.6" + }, + { + "name":"harfbuzz", + "git-repo":"https://github.com/harfbuzz/harfbuzz", + "git-ref":"8.1.1" + }, + { + "name":"zlib", + "git-repo":"https://github.com/intel/zlib", + "git-ref":"v1.2.13_jtk" + }, + { + "name":"libpng", + "git-repo":"https://github.com/glennrp/libpng", + "git-ref":"v1.6.40" + }, + { + "name":"bzip2", + "git-repo":"https://gitlab.com/bzip2/bzip2.git", + "git-ref":"bzip2-1.0.8" + }, + { + "name":"freetype", + "git-repo":"https://gitlab.freedesktop.org/freetype/freetype/", + "git-ref":"VER-2-13-1" + }, + { + "name":"tcl", + "git-repo":"https://github.com/tcltk/tcl", + "git-ref":"core-8-6-13" + }, + { + "name":"tk", + "git-repo":"https://github.com/tcltk/tk", + "git-ref":"core-8-6-13" + }, + { + "name":"opencascade", + "git-repo":"https://gitlab.com/blobfish/occt", + "git-ref":"V7_7_1_BF" + }, + { + "name":"netgen", + "git-repo":"https://github.com/NGSolve/netgen", + "git-ref":"v6.2.2304" + }, + { + "name":"hdf5", + "git-repo":"https://github.com/HDFGroup/hdf5", + "git-ref":"hdf5-1_10_8" + }, + { + "name":"medfile", + "url":"http://files.salome-platform.org/Salome/other/med-4.1.1.tar.gz" + }, + { + "name":"gmsh", + "git-repo":"https://gitlab.onelab.info/gmsh/gmsh", + "git-ref":"gmsh_4_11_1" + }, + { + "name":"pycxx", + "git-repo":"https://github.com/montylab3d/pycxx", + "git-ref":"7.1.5" + }, + { + "name":"icu", + "git-repo":"https://github.com/unicode-org/icu", + "git-ref":"release-73-2" + }, + { + "name":"xerces-c", + "git-repo":"https://github.com/apache/xerces-c", + "git-ref":"v3.2.4" + }, + { + "name":"libfmt", + "git-repo":"https://github.com/fmtlib/fmt", + "git-ref":"10.1.0" + }, + { + "name":"eigen3", + "git-repo":"https://gitlab.com/libeigen/eigen", + "git-ref":"3.4.0" + } + ] +} diff --git a/create_libpack.py b/create_libpack.py new file mode 100644 index 0000000..3186e4c --- /dev/null +++ b/create_libpack.py @@ -0,0 +1,90 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +# Prerequisites: +# * A working compiler toolchain for your system, accessible by cMake +# * CMake +# * git +# * 7-zip +# * Some version of Python that can run this file +# * Network access +# * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF + +import argparse +import json +import os +import shutil +import subprocess + +CONFIG = {} + +def delete_existing(path:str, silent:bool=False): + """ Delete a directory tree, with optional confirmation sequence """ + do_it = silent + if not silent and os.path.exists(path): + response = input(f"Really delete entire path {path}? y/N ") + if response.lower() == "y": + do_it = True + if do_it: + print("Removing {path} prior to beginning") + shutil.rmtree(path) + else: + print("NOT removing {path}") + +def load_config(path:str): + """ Load a JSON-formatted configuration file for this utility """ + with open (path, "r", encoding="utf-8") as f: + config_data = f.read() + CONFIG = json.loads(config_data) + +def clone_git_repos(): + """ Clone the required repos """ + content = CONFIG["content"] + for item in content: + if "git-repo" in item and "git-ref" in item: + clone(item["git-repo"], item["git-ref"]) + +def clone(url:str, ref:str): + """ Shallow clones a git repo at the given ref using a system-installed git """ + try: + run_result = subprocess.run([ + "git", + "clone", + "--branch", + ref, + "--depth", + "1", + url, + ], capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: FAILED TO CLONE REPO {url} AT REF {ref}") + print(e.output) + exit(e.returncode) + +def download(url:str): + """ Directly downloads some sort of compressed format file and decompresses it using a system-installed 7-zip""" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Builds a collection of FreeCAD dependencies for the current system" + ) + parser.add_argument("-c", "--config", help="Path to a JSON configuration file for this utility", default="./config.json") + parser.add_argument("-w", "--working", help="Directory to put all the clones and downloads in", default="./working") + parser.add_argument("-e", "--skip-existing-clone", action='store_true', help="If a given clone (or download) directory exists, skip cloning/downloading") + parser.add_argument("-b", "--skip-existing-build", action='store_true', help="If a given build directory exists, skip building") + parser.add_argument("-s", "--silent", action='store_true', help="I kow what I'm doing, don't ask me any questions") + parser.add_argument("path-to-final-libpack-dir", nargs='?', default="./") + args = vars(parser.parse_args()) + + if not args["skip_existing_clone"]: + delete_existing(args["working"], silent=args["silent"]) + + os.makedirs(args["working"], exist_ok=True) + os.chdir(args["working"]) + load_config(args["config"]) + + clone_git_repos() + + From f00f07e4a1f203b57dd642586d036967857f2f6f Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 18 Aug 2023 21:14:21 -0500 Subject: [PATCH 02/27] Create initial build script. Incomplete --- .gitignore | 3 + compile_all.py | 73 +++++++++++++ config.json | 12 ++- create_libpack.py | 187 ++++++++++++++++++++++++--------- test_compile_all.py | 72 +++++++++++++ test_create_libpack.py | 232 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 527 insertions(+), 52 deletions(-) create mode 100644 .gitignore create mode 100644 compile_all.py create mode 100644 test_compile_all.py create mode 100644 test_create_libpack.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d1805d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__ +.vscode/ +working diff --git a/compile_all.py b/compile_all.py new file mode 100644 index 0000000..125a79d --- /dev/null +++ b/compile_all.py @@ -0,0 +1,73 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +# Every package has its own compilation and installation idiosyncrasies, so we have to use a custom +# build script for each one. + +from enum import Enum +import os +import subprocess +import sys + + +class build_mode(Enum): + DEBUG = 1 + RELEASE = 2 + + +def compile_all(config: dict, mode: build_mode): + content = config["content"] + base_dir = os.curdir + create_libpack_dir(config, mode) + for item in content: + # All build methods are named using "build_XXX" where XXX is the name of the package in the config file + os.chdir(item["name"]) + build_function_name = "build_" + item["name"] + build_function = globals()[build_function_name] + build_function(mode) + os.chdir(base_dir) + + +def create_libpack_dir(config: dict, mode: build_mode) -> str: + """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of + the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. + """ + + dirname = "LibPack-{}-v{}-{}".format( + config["FreeCAD-version"], + config["LibPack-version"], + "release" if mode == build_mode.RELEASE else "debug", + ) + if os.path.exists(dirname): + backup_name = dirname + "-backup-" + "a" + while os.path.exists(backup_name): + if backup_name[-1] =="z": + print( + "You have too many old LibPack backup directories. Please delete some of them." + ) + exit(1) + backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) + + os.rename(dirname, backup_name) + os.mkdir(dirname) + return dirname + +def build_nonexistent(mode: build_mode): + """ Used for automated testing to allow Mock injection """ + pass + +def build_python(mode: build_mode): + if sys.platform.startswith("win32"): + subprocess.run( + [ + "PCbuild\\build.bat", + "-p", + "x64", + "-c", + "Release" if mode == build_mode.RELEASE else "Debug", + ], + check=True, + ) + else: + raise NotImplemented("Non-windows compilation of Python is not implemented yet") diff --git a/config.json b/config.json index ddaf4be..b3d27f5 100644 --- a/config.json +++ b/config.json @@ -1,7 +1,7 @@ { "FreeCAD-version":"0.22", "LibPack-version":"3.0.0", - "contents": [ + "content": [ { "name":"python", "git-repo":"https://github.com/python/cpython", @@ -26,9 +26,15 @@ "git-repo":"https://github.com/coin3d/quarter", "git-ref":"master" }, + { + "name":"pcre2", + "git-repo":"https://github.com/PCRE2Project/pcre2", + "git-ref":"pcre2-10.42" + }, { "name":"swig", - "url":"http://prdownloads.sourceforge.net/swig/swigwin-4.1.1.zip" + "git-repo":"https://github.com/swig/swig.git", + "git-tag":"v4.1.1" }, { "name":"pivy", @@ -100,7 +106,7 @@ }, { "name":"medfile", - "url":"http://files.salome-platform.org/Salome/other/med-4.1.1.tar.gz" + "git-repo":"https://github.com/FedoraScientific/salome-med" }, { "name":"gmsh", diff --git a/create_libpack.py b/create_libpack.py index 3186e4c..0e7ae82 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -3,88 +3,177 @@ # SPDX-License-Identifier: LGPL-2.1-or-later # Prerequisites: +# * Network access # * A working compiler toolchain for your system, accessible by cMake -# * CMake +# * CMake # * git -# * 7-zip +# * 7z (see https://www.7-zip.org) # * Some version of Python that can run this file -# * Network access +# * The "requests" Python package (e.g. 'pip install requests') # * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF +# * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) import argparse import json import os import shutil +import stat import subprocess +import requests +from urllib.parse import urlparse + +import compile_all + +path_to_7zip = "C:\\Program Files\\7-Zip\\7z.exe" +path_to_bison = "C:\\Program Files\\win_flex_bison\\win_bison.exe" + + +def remove_readonly(func, path, _) -> None: + """Remove a read-only file.""" + + os.chmod(path, stat.S_IWRITE) + func(path) -CONFIG = {} - -def delete_existing(path:str, silent:bool=False): - """ Delete a directory tree, with optional confirmation sequence """ - do_it = silent - if not silent and os.path.exists(path): - response = input(f"Really delete entire path {path}? y/N ") - if response.lower() == "y": - do_it = True - if do_it: - print("Removing {path} prior to beginning") - shutil.rmtree(path) - else: - print("NOT removing {path}") - -def load_config(path:str): - """ Load a JSON-formatted configuration file for this utility """ - with open (path, "r", encoding="utf-8") as f: + +def delete_existing(path: str, silent: bool = False): + """Delete a directory tree, with optional confirmation sequence""" + if os.path.exists(path): + if not silent: + response = input(f"Really delete entire path {path}? y/N ") + if response.lower() != "y": + print(f"NOT removing {path}") + return + print(f"Removing {path} prior to beginning") + shutil.rmtree(path, onerror=remove_readonly) + + +def load_config(path: str) -> dict: + """Load a JSON-formatted configuration file for this utility""" + if not os.path.exists(path): + print(f"ERROR: No such config file '{path}'") + exit(1) + with open(path, "r", encoding="utf-8") as f: config_data = f.read() - CONFIG = json.loads(config_data) + try: + return json.loads(config_data) + except json.JSONDecodeError: + print("ERROR: The config file does not contain valid JSON data") + exit(1) -def clone_git_repos(): - """ Clone the required repos """ - content = CONFIG["content"] + +def fetch_remote_data(config: dict, skip_existing: bool = False): + """Clone the required repos and download the URLs""" + content = config["content"] for item in content: + if skip_existing and os.path.exists(item["name"]): + continue if "git-repo" in item and "git-ref" in item: - clone(item["git-repo"], item["git-ref"]) - -def clone(url:str, ref:str): - """ Shallow clones a git repo at the given ref using a system-installed git """ + clone(item["name"], item["git-repo"], item["git-ref"]) + elif "git-repo" in item: + clone(item["name"], item["git-repo"]) + elif "git-ref" in item: + print(f"ERROR: found a git ref without a git repo for {item['name']}") + exit() + elif "url" in item: + download(item["name"], item["url"]) + + +def clone(name: str, url: str, ref: str = None): + """Shallow clones a git repo at the given ref using a system-installed git""" try: - run_result = subprocess.run([ - "git", - "clone", - "--branch", - ref, - "--depth", - "1", - url, - ], capture_output=True) + if ref is None: + print(f"Cloning {url}") + else: + print(f"Cloning {url} at {ref}") + args = ["git", "clone"] + if ref is not None: + args.extend(["--branch", ref]) + args.extend(["--depth", "1", "--recurse-submodules", url, name]) + subprocess.run(args, capture_output=True, check=True) except subprocess.CalledProcessError as e: - print("ERROR: FAILED TO CLONE REPO {url} AT REF {ref}") + print("ERROR: failed to clone git repo {url} at ref {ref}") print(e.output) exit(e.returncode) -def download(url:str): - """ Directly downloads some sort of compressed format file and decompresses it using a system-installed 7-zip""" + +def download(name: str, url: str): + """Directly downloads some sort of compressed format file and decompresses it using a system-installed 7-zip""" + print(f"Downloading {name} from {url}") + os.mkdir(name) + request_result = requests.get(url) + parsed_url = urlparse(url) + filename = parsed_url.path.rsplit("/", 1)[-1] + with open(os.path.join(name, filename), "wb") as f: + f.write(request_result.content) + decompress(name, filename) + + +def decompress(name: str, filename: str): + original_dir = os.getcwd() + os.chdir(name) + try: + subprocess.run([path_to_7zip, "x", filename], capture_output=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: failed to unzip {filename} at from {name} using {path_to_7zip}") + print(e.output) + exit(e.returncode) + os.chdir(original_dir) if __name__ == "__main__": parser = argparse.ArgumentParser( description="Builds a collection of FreeCAD dependencies for the current system" ) - parser.add_argument("-c", "--config", help="Path to a JSON configuration file for this utility", default="./config.json") - parser.add_argument("-w", "--working", help="Directory to put all the clones and downloads in", default="./working") - parser.add_argument("-e", "--skip-existing-clone", action='store_true', help="If a given clone (or download) directory exists, skip cloning/downloading") - parser.add_argument("-b", "--skip-existing-build", action='store_true', help="If a given build directory exists, skip building") - parser.add_argument("-s", "--silent", action='store_true', help="I kow what I'm doing, don't ask me any questions") - parser.add_argument("path-to-final-libpack-dir", nargs='?', default="./") + parser.add_argument( + "-c", + "--config", + help="Path to a JSON configuration file for this utility", + default="./config.json", + ) + parser.add_argument( + "-w", + "--working", + help="Directory to put all the clones and downloads in", + default="./working", + ) + parser.add_argument( + "-e", + "--skip-existing-clone", + action="store_true", + help="If a given clone (or download) directory exists, skip cloning/downloading", + ) + parser.add_argument( + "-b", + "--skip-existing-build", + action="store_true", + help="If a given build directory exists, skip building", + ) + parser.add_argument( + "-s", + "--silent", + action="store_true", + help="I kow what I'm doing, don't ask me any questions", + ) + parser.add_argument("--7zip", help="Path to 7-zip executable", default=path_to_7zip) + parser.add_argument( + "--bison", help="Path to Bison executable", default=path_to_bison + ) + parser.add_argument("path-to-final-libpack-dir", nargs="?", default="./") args = vars(parser.parse_args()) + config = load_config(args["config"]) + path_to_7zip = args["7zip"] + path_to_bison = args["bison"] + if not args["skip_existing_clone"]: delete_existing(args["working"], silent=args["silent"]) os.makedirs(args["working"], exist_ok=True) os.chdir(args["working"]) - load_config(args["config"]) - clone_git_repos() + fetch_remote_data(config, args["skip_existing_clone"]) +# Preliminary setup that will be needed for running CMake +# CMAKE_PREFIX_PATH to libpack dir +# CMAKE_INSTALL_PREFIX to libpack dir diff --git a/test_compile_all.py b/test_compile_all.py new file mode 100644 index 0000000..d5c0e4e --- /dev/null +++ b/test_compile_all.py @@ -0,0 +1,72 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +import os +import tempfile +import unittest +from unittest.mock import MagicMock, mock_open, patch + +import compile_all + + +class TestCompileAll(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.original_dir = os.getcwd() + + def tearDown(self) -> None: + os.chdir(self.original_dir) + super().tearDown() + + @patch("compile_all.create_libpack_dir") + @patch("os.chdir") + @patch("compile_all.build_nonexistent") + def test_compile_all_calls_build_function(self, nonexistent_mock:MagicMock, _1, _2): + config = {"content":[ + {"name":"nonexistent"} + ]} + compile_all.compile_all(config, mode=compile_all.build_mode.RELEASE) + nonexistent_mock.assert_called_once() + + def test_create_libpack_dir_no_conflict(self): + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} + dirname = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + self.assertNotEqual(dirname.find("0.22"), -1) + self.assertNotEqual(dirname.find("3.0.0"), -1) + self.assertNotEqual(dirname.find("release"), -1) + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + def test_create_libpack_dir_needs_backup(self): + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} + first = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + second = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + self.assertTrue(os.path.exists(second)) + self.assertEqual(len(os.listdir()), 2) + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + def test_too_many_backups_exits(self): + with tempfile.TemporaryDirectory() as temp_dir: + try: + # Arrange + os.chdir(temp_dir) + config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} + for i in range(0,27): + compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + self.assertEqual(len(os.listdir()), i+1) + # Act + with self.assertRaises(SystemExit): + compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + except: + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + raise + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + + +if __name__ == "__main__": + unittest.main() diff --git a/test_create_libpack.py b/test_create_libpack.py new file mode 100644 index 0000000..9745522 --- /dev/null +++ b/test_create_libpack.py @@ -0,0 +1,232 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +import os +import shutil +from subprocess import CalledProcessError +import tempfile +import unittest +from unittest.mock import MagicMock, patch, mock_open + +import create_libpack + + +class TestDeleteExisting(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self) -> None: + super().tearDown() + shutil.rmtree(self.temp_dir.name) + + @patch("builtins.print") + def test_no_directory_not_silent(self, mock_print: MagicMock): + """Nothing happens when asking to delete a directory that does not exist""" + create_libpack.delete_existing( + os.path.join(self.temp_dir.name, "no_such_dir"), silent=False + ) + mock_print.assert_not_called() + + @patch("builtins.print") + def test_with_directory_silent_is_silent(self, mock_print: MagicMock): + """In silent mode, nothing is printed even when deleting""" + dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") + os.mkdir(dir_to_delete) + create_libpack.delete_existing(dir_to_delete, silent=True) + mock_print.assert_not_called() + + @patch("builtins.print") + def test_with_directory_silent_deletes_dir(self, mock_print: MagicMock): + """In silent mode, the directory is deleted""" + dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") + os.mkdir(dir_to_delete) + create_libpack.delete_existing(dir_to_delete, silent=True) + self.assertFalse(os.path.exists(dir_to_delete)) + + @patch("builtins.input") + def test_with_directory_not_silent_asks_for_confirmation( + self, mock_input: MagicMock + ): + """When not in silent mode, the user is asked to confirm""" + dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") + os.mkdir(dir_to_delete) + create_libpack.delete_existing(dir_to_delete, silent=False) + mock_input.assert_called_once() + + @patch("builtins.input") + def test_confirm_defaults_to_no(self, mock_input: MagicMock): + """If the user just hits enter, the default is to NOT delete the directory""" + dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") + mock_input.return_value = "" + os.mkdir(dir_to_delete) + create_libpack.delete_existing(dir_to_delete, silent=False) + self.assertTrue(os.path.exists(dir_to_delete)) + + @patch("builtins.input") + def test_confirm_with_y_deletes(self, mock_input: MagicMock): + """If the user types 'y' then the directory is deleted""" + dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") + mock_input.return_value = "y" + os.mkdir(dir_to_delete) + create_libpack.delete_existing(dir_to_delete, silent=False) + self.assertFalse(os.path.exists(dir_to_delete)) + + +class TestLoadConfig(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self) -> None: + super().tearDown() + shutil.rmtree(self.temp_dir.name) + + @patch("builtins.open", mock_open(read_data='{"entry1":1,"entry2":2}')) + def test_json_is_loaded(self): + """When appropriate JSON data exists it is loaded and returned""" + loaded_data = create_libpack.load_config(self.temp_dir.name) + self.assertIn("entry1", loaded_data) + self.assertIn("entry2", loaded_data) + + @patch("builtins.print") + def test_non_existent_file_prints_error(self, mock_print: MagicMock): + """If a non-existent file is given, an error is printed (and exit() is called)""" + with self.assertRaises(SystemExit): + create_libpack.load_config( + os.path.join(self.temp_dir.name, "no_such_file.json") + ) + mock_print.assert_called_once() + + @patch("builtins.print") + @patch("builtins.open", mock_open(read_data="bad json data!")) + def test_bad_file_prints_error(self, mock_print: MagicMock): + """If a bad JSON data is given, an error is printed (and exit() is called)""" + with self.assertRaises(SystemExit): + create_libpack.load_config(self.temp_dir.name) + mock_print.assert_called_once() + + +class TestRemoteFetchFunctions(unittest.TestCase): + """Git and direct download""" + + def setUp(self) -> None: + super().setUp() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self) -> None: + super().tearDown() + shutil.rmtree(self.temp_dir.name) + + @patch("create_libpack.clone") + def test_repos_are_discovered(self, mock_clone: MagicMock): + """Any dictionary with both a git-repo and a git-ref is passed along to clone""" + test_config = { + "content": [ + {"name": "test1", "git-repo": "test1_repo", "git-ref": "test1_ref"}, + {"name": "test2", "git-repo": "test2_repo", "git-ref": "test2_ref"}, + {"name": "test3", "git-repo": "test3_repo", "git-ref": "test3_ref"}, + ] + } + create_libpack.fetch_remote_data(test_config) + self.assertEqual(mock_clone.call_count, 3) + + @patch("builtins.print") + def test_missing_repo_errors_if_ref(self, mock_print: MagicMock): + """An entry with a git-ref but no git-repo is an error""" + test_config = {"content": [{"name": "test1", "git-ref": "test1_ref"}]} + with self.assertRaises(SystemExit): + create_libpack.fetch_remote_data(test_config) + mock_print.assert_called() + + @patch("create_libpack.clone") + def test_missing_ref_is_omitted(self, mock_clone: MagicMock): + """An entry with a git-repo but no git-ref just doesn't use the ref""" + test_config = { + "content": [ + {"name": "test1", "git-repo": "test1_repo"}, + ] + } + create_libpack.fetch_remote_data(test_config) + mock_clone.assert_called_once_with("test1", "test1_repo") + + @patch("create_libpack.clone") + def test_non_git_entries_are_ignored(self, mock_clone: MagicMock): + """Non-git entries are just ignored""" + test_config = { + "content": [ + {"name": "test1"}, + ] + } + create_libpack.fetch_remote_data(test_config) + mock_clone.assert_not_called() + + @patch("subprocess.run") + def test_clone_calls_git_with_ref(self, run_mock: MagicMock): + """When given a repo and a ref, git clone is set up appropriately""" + create_libpack.clone("name", "https://some.url", "some_git_ref") + run_mock.assert_called_once() + call_data: list = run_mock.call_args[0][0] + self.assertIn("https://some.url", call_data) + self.assertIn("some_git_ref", call_data) + self.assertEquals(call_data[-1], "name") + + @patch("subprocess.run") + def test_clone_calls_git_without_ref(self, run_mock: MagicMock): + """When given a repo and a ref, git clone is set up appropriately""" + create_libpack.clone("test", "https://some.url") + run_mock.assert_called_once() + call_data = run_mock.call_args[0][0] + self.assertNotIn(None, call_data) + self.assertNotIn("--branch", call_data) + + @patch("subprocess.run") + def test_exception_is_caught_and_calls_exit(self, run_mock: MagicMock): + """When given a repo and a ref, git clone is set up appropriately""" + run_mock.side_effect = CalledProcessError(1, "command_that_was_called") + with self.assertRaises(SystemExit): + create_libpack.clone("some_name", "https://some.url") + + @patch("os.path.exists", MagicMock(return_value=True)) + @patch("create_libpack.clone") + def test_skips_existing_paths_with_flag(self, clone_mock: MagicMock): + test_config = { + "content": [ + {"name": "test1", "git-repo": "test1_repo", "git-ref": "test1_ref"}, + {"name": "test2", "git-repo": "test2_repo", "git-ref": "test2_ref"}, + {"name": "test3", "git-repo": "test3_repo", "git-ref": "test3_ref"}, + ] + } + create_libpack.fetch_remote_data(test_config, skip_existing=True) + clone_mock.assert_not_called() + + @patch("create_libpack.download") + def test_url_calls_download(self, download_mock: MagicMock): + test_config = {"content": [{"name": "test", "url": "https://some.url"}]} + create_libpack.fetch_remote_data(test_config) + download_mock.assert_called_once() + + @patch("os.mkdir") # Patch so it doesn't actually make a directory + @patch("requests.get") # Patch so no network request is made + @patch("create_libpack.decompress") # Patch so no attempt is made to decompress + def test_download_creates_file(self, decompress_mock: MagicMock, _1, _2): + with patch("builtins.open", mock_open()) as open_mock: + create_libpack.download("make_this_dir", "https://some.url/test.7z") + open_mock.assert_called_once_with( + os.path.join("make_this_dir", "test.7z"), "wb" + ) + decompress_mock.assert_called_once_with("make_this_dir", "test.7z") + + @patch("os.chdir") + @patch("subprocess.run") + def test_decompress_calls_subprocess( + self, run_mock: MagicMock, chdir_mock: MagicMock + ): + create_libpack.decompress("path_to_file", "file_name") + run_mock.assert_called_once() + self.assertEqual(chdir_mock.call_count, 2) + + +if __name__ == "__main__": + unittest.main() From 9a268d54f5e2bdeaa66da6a1a42e5854a8617884 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 19 Aug 2023 13:00:25 -0500 Subject: [PATCH 03/27] Add Qt copy and start Boost compile --- compile_all.py | 163 +++++++++++++++++++++++++++++--------------- create_libpack.py | 13 +++- test_compile_all.py | 26 ++++--- 3 files changed, 136 insertions(+), 66 deletions(-) diff --git a/compile_all.py b/compile_all.py index 125a79d..d513c86 100644 --- a/compile_all.py +++ b/compile_all.py @@ -7,6 +7,8 @@ from enum import Enum import os +import platform +import shutil import subprocess import sys @@ -15,59 +17,112 @@ class build_mode(Enum): DEBUG = 1 RELEASE = 2 + def __str__(self) -> str: + if self == build_mode.DEBUG: + return "Debug" + elif self == build_mode.RELEASE: + return "Release" + else: + return "Unknown" -def compile_all(config: dict, mode: build_mode): - content = config["content"] - base_dir = os.curdir - create_libpack_dir(config, mode) - for item in content: - # All build methods are named using "build_XXX" where XXX is the name of the package in the config file - os.chdir(item["name"]) - build_function_name = "build_" + item["name"] - build_function = globals()[build_function_name] - build_function(mode) - os.chdir(base_dir) - - -def create_libpack_dir(config: dict, mode: build_mode) -> str: - """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of - the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. - """ - - dirname = "LibPack-{}-v{}-{}".format( - config["FreeCAD-version"], - config["LibPack-version"], - "release" if mode == build_mode.RELEASE else "debug", - ) - if os.path.exists(dirname): - backup_name = dirname + "-backup-" + "a" - while os.path.exists(backup_name): - if backup_name[-1] =="z": - print( - "You have too many old LibPack backup directories. Please delete some of them." - ) - exit(1) - backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) - - os.rename(dirname, backup_name) - os.mkdir(dirname) - return dirname - -def build_nonexistent(mode: build_mode): - """ Used for automated testing to allow Mock injection """ - pass - -def build_python(mode: build_mode): - if sys.platform.startswith("win32"): - subprocess.run( - [ - "PCbuild\\build.bat", - "-p", - "x64", - "-c", - "Release" if mode == build_mode.RELEASE else "Debug", - ], - check=True, + +class Compiler: + def __init__(self, config, mode, bison_path, skip_existing:bool=False): + self.config = config + self.mode = mode + self.bison_path = bison_path + self.base_dir = os.getcwd() + self.skip_existing = skip_existing + + def compile_all(self): + content = self.config["content"] + libpack_dir = self.create_libpack_dir() + self.install_dir = os.path.join(os.getcwd(), libpack_dir) + for item in content: + # All build methods are named using "build_XXX" where XXX is the name of the package in the config file + print(f"Building {item['name']} in {self.mode} mode") + os.chdir(item["name"]) + build_function_name = "build_" + item["name"] + build_function = getattr(self, build_function_name) + build_function(item) + os.chdir(self.base_dir) + + def create_libpack_dir(self) -> str: + """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of + the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. + """ + + dirname = "LibPack-{}-v{}-{}".format( + self.config["FreeCAD-version"], + self.config["LibPack-version"], + str(self.mode), ) - else: - raise NotImplemented("Non-windows compilation of Python is not implemented yet") + if os.path.exists(dirname) and not self.skip_existing: + backup_name = dirname + "-backup-" + "a" + while os.path.exists(backup_name): + if backup_name[-1] == "z": + print( + "You have too many old LibPack backup directories. Please delete some of them." + ) + exit(1) + backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) + + os.rename(dirname, backup_name) + if not os.path.exists(dirname): + os.mkdir(dirname) + return dirname + + def build_nonexistent(self, options=None): + """Used for automated testing to allow easy Mock injection""" + + def build_python(self, options=None): + if sys.platform.startswith("win32"): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir,"bin","python.exe")): + print("Skipping existing Python") + return + try: + arch = "x64" if platform.machine() == "AMD64" else "ARM64" + path = "amd64" if platform.machine() == "AMD64" else "arm64" + subprocess.run( + [ + "PCbuild\\build.bat", + "-p", + arch, + "-c", + str(self.mode), + ], + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + print("Python build failed") + print(e.output) + exit(e.returncode) + bin_dir = os.path.join(self.install_dir, "bin") + os.makedirs(bin_dir, exist_ok=True) + shutil.copytree(f"PCBuild\\{path}", bin_dir, dirs_exist_ok=True) + else: + raise NotImplemented( + "Non-windows compilation of Python is not implemented yet" + ) + + def build_qt(self, options:dict): + """Doesn't really "build" Qt, just copies the pre-compiled libraries from the configured path""" + qt_dir = options["install-directory"] + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir,"metatypes")): + print("Skipping existing Qt") + return + if not os.path.exists(qt_dir): + print(f"Error: specified Qt installation path does not exist ({qt_dir})") + exit(1) + shutil.copytree(qt_dir, self.install_dir, dirs_exist_ok=True) + + def build_boost(self, options:dict=None): + """ Builds boost shared libraries and installs libraries and headers """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir,"include","boost")): + print("Skipping existing boost") + return + # Boost uses a custom build system and needs a config file \ No newline at end of file diff --git a/create_libpack.py b/create_libpack.py index 0e7ae82..58a4294 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -76,6 +76,9 @@ def fetch_remote_data(config: dict, skip_existing: bool = False): exit() elif "url" in item: download(item["name"], item["url"]) + else: + # Just make the directory, presumably later code will know what to do + os.makedirs(item["name"], exist_ok=True) def clone(name: str, url: str, ref: str = None): @@ -91,7 +94,7 @@ def clone(name: str, url: str, ref: str = None): args.extend(["--depth", "1", "--recurse-submodules", url, name]) subprocess.run(args, capture_output=True, check=True) except subprocess.CalledProcessError as e: - print("ERROR: failed to clone git repo {url} at ref {ref}") + print(f"ERROR: failed to clone git repo {url} at ref {ref}") print(e.output) exit(e.returncode) @@ -173,6 +176,14 @@ def decompress(name: str, filename: str): fetch_remote_data(config, args["skip_existing_clone"]) + compiler = compile_all.Compiler( + config, + compile_all.build_mode.RELEASE, + bison_path=path_to_bison, + skip_existing=args["skip_existing_build"], + ) + compiler.compile_all() + # Preliminary setup that will be needed for running CMake # CMAKE_PREFIX_PATH to libpack dir diff --git a/test_compile_all.py b/test_compile_all.py index d5c0e4e..3038a2f 100644 --- a/test_compile_all.py +++ b/test_compile_all.py @@ -13,27 +13,32 @@ class TestCompileAll(unittest.TestCase): def setUp(self) -> None: super().setUp() + config = {"FreeCAD-version": "0.22", + "LibPack-version": "3.0.0", + "content":[ + {"name":"nonexistent"} + ]} + self.compiler = compile_all.Compiler(config, compile_all.build_mode.RELEASE, "bison_path") self.original_dir = os.getcwd() def tearDown(self) -> None: os.chdir(self.original_dir) super().tearDown() - @patch("compile_all.create_libpack_dir") + @patch("compile_all.Compiler.create_libpack_dir") @patch("os.chdir") - @patch("compile_all.build_nonexistent") + @patch("compile_all.Compiler.build_nonexistent") def test_compile_all_calls_build_function(self, nonexistent_mock:MagicMock, _1, _2): config = {"content":[ {"name":"nonexistent"} ]} - compile_all.compile_all(config, mode=compile_all.build_mode.RELEASE) + self.compiler.compile_all() nonexistent_mock.assert_called_once() def test_create_libpack_dir_no_conflict(self): with tempfile.TemporaryDirectory() as temp_dir: os.chdir(temp_dir) - config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} - dirname = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + dirname = self.compiler.create_libpack_dir() self.assertNotEqual(dirname.find("0.22"), -1) self.assertNotEqual(dirname.find("3.0.0"), -1) self.assertNotEqual(dirname.find("release"), -1) @@ -42,25 +47,24 @@ def test_create_libpack_dir_no_conflict(self): def test_create_libpack_dir_needs_backup(self): with tempfile.TemporaryDirectory() as temp_dir: os.chdir(temp_dir) - config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} - first = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) - second = compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + first = self.compiler.create_libpack_dir() + second = self.compiler.create_libpack_dir() self.assertTrue(os.path.exists(second)) self.assertEqual(len(os.listdir()), 2) os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - def test_too_many_backups_exits(self): + def test_too_many_backups_exist(self): with tempfile.TemporaryDirectory() as temp_dir: try: # Arrange os.chdir(temp_dir) config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} for i in range(0,27): - compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + self.compiler.create_libpack_dir() self.assertEqual(len(os.listdir()), i+1) # Act with self.assertRaises(SystemExit): - compile_all.create_libpack_dir(config, mode=compile_all.build_mode.RELEASE) + self.compiler.create_libpack_dir() except: os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows raise From b4bfd7eedf803b7c0f7fd26654a74d49d48fac8b Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 19 Aug 2023 17:18:03 -0500 Subject: [PATCH 04/27] Reconfigure to get boost-python working --- bootstrap.py | 73 ++++++++++++++++++++++++++++ compile_all.py | 115 ++++++++++++++++++++++++++------------------ config.json | 5 -- create_libpack.py | 2 +- test_bootstrap.py | 72 +++++++++++++++++++++++++++ test_compile_all.py | 65 +++++++++---------------- 6 files changed, 235 insertions(+), 97 deletions(-) create mode 100644 bootstrap.py create mode 100644 test_bootstrap.py diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 0000000..f4ba4be --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,73 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +import json +import os +import sys + +from compile_all import BuildMode + +""" """ + +def parse_args() -> dict: + if len(sys.argv) > 3: + usage() + exit(1) + new_config_dict = {"mode": BuildMode.RELEASE, "config_file": "config.json"} + for arg in sys.argv[1:]: + key, value = extract_arg(arg) + new_config_dict[key] = value + return new_config_dict + + +def extract_arg(arg) -> tuple[str, object]: + if arg.lower() in ["release", "debug"]: + return "mode", BuildMode.RELEASE if arg.lower() == "release" else BuildMode.DEBUG + return "config_file", arg + + +def usage(): + print("Used to create the base LibPack directory that you will then manually install Python into") + print("Usage: python bootstrap.py [config_file] [release|debug]") + print() + print('Result: A new working/LibPack-XX-YY-MM directory has been created') + print('Next step: install Python into working/LibPack-XX-YY-MM/bin, then run:') + print(' .\\working\\LibPack-XX-YY-MM\\bin\\python create_libpack.py') + print('(where XX, YY, and MM change according to the config and inputs)') + + +def create_libpack_dir(config: dict, mode: BuildMode) -> str: + """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of + the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. + """ + + dirname = "LibPack-{}-v{}-{}".format( + config["FreeCAD-version"], + config["LibPack-version"], + str(mode), + ) + if os.path.exists(dirname): + backup_name = dirname + "-backup-" + "a" + while os.path.exists(backup_name): + if backup_name[-1] == "z": + print( + "You have too many old LibPack backup directories. Please delete some of them." + ) + exit(1) + backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) + + os.rename(dirname, backup_name) + if not os.path.exists(dirname): + os.mkdir(dirname) + return dirname + + +if __name__ == "__main__": + args = parse_args() + with open(args["config_file"], "r", encoding="utf-8") as f: + config_data = f.read() + config_dict = json.load(config_data) + os.makedirs("working", exist_ok=True) + os.chdir("working") + create_libpack_dir(config_dict, args["mode"]) diff --git a/compile_all.py b/compile_all.py index d513c86..2deb25b 100644 --- a/compile_all.py +++ b/compile_all.py @@ -13,32 +13,35 @@ import sys -class build_mode(Enum): +class BuildMode(Enum): DEBUG = 1 RELEASE = 2 def __str__(self) -> str: - if self == build_mode.DEBUG: + if self == BuildMode.DEBUG: return "Debug" - elif self == build_mode.RELEASE: + elif self == BuildMode.RELEASE: return "Release" else: return "Unknown" class Compiler: - def __init__(self, config, mode, bison_path, skip_existing:bool=False): + def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.config = config self.mode = mode self.bison_path = bison_path self.base_dir = os.getcwd() self.skip_existing = skip_existing + libpack_dir = "LibPack-{}-v{}-{}".format( + config["FreeCAD-version"], + config["LibPack-version"], + str(mode), + ) + self.install_dir = os.path.join(os.getcwd(), libpack_dir) def compile_all(self): - content = self.config["content"] - libpack_dir = self.create_libpack_dir() - self.install_dir = os.path.join(os.getcwd(), libpack_dir) - for item in content: + for item in self.config["content"]: # All build methods are named using "build_XXX" where XXX is the name of the package in the config file print(f"Building {item['name']} in {self.mode} mode") os.chdir(item["name"]) @@ -47,40 +50,16 @@ def compile_all(self): build_function(item) os.chdir(self.base_dir) - def create_libpack_dir(self) -> str: - """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of - the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. - """ - - dirname = "LibPack-{}-v{}-{}".format( - self.config["FreeCAD-version"], - self.config["LibPack-version"], - str(self.mode), - ) - if os.path.exists(dirname) and not self.skip_existing: - backup_name = dirname + "-backup-" + "a" - while os.path.exists(backup_name): - if backup_name[-1] == "z": - print( - "You have too many old LibPack backup directories. Please delete some of them." - ) - exit(1) - backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) - - os.rename(dirname, backup_name) - if not os.path.exists(dirname): - os.mkdir(dirname) - return dirname - - def build_nonexistent(self, options=None): + def build_nonexistent(self, _=None): """Used for automated testing to allow easy Mock injection""" - def build_python(self, options=None): + def build_python(self, _=None): + """ NOTE: This doesn't install correctly, so should not be used at this time... install Python manually """ if sys.platform.startswith("win32"): - if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir,"bin","python.exe")): - print("Skipping existing Python") - return + expected_exe_path = os.path.join(self.install_dir, "bin", "python.exe") + if self.skip_existing and os.path.exists(expected_exe_path): + print("Not rebuilding, instead just using existing Python in the LibPack installation path") + return try: arch = "x64" if platform.machine() == "AMD64" else "ARM64" path = "amd64" if platform.machine() == "AMD64" else "arm64" @@ -97,32 +76,72 @@ def build_python(self, options=None): ) except subprocess.CalledProcessError as e: print("Python build failed") - print(e.output) + print(e.output.decode("utf-8")) exit(e.returncode) bin_dir = os.path.join(self.install_dir, "bin") + lib_dir = os.path.join(bin_dir, "Lib") + inc_dir = os.path.join(bin_dir, "Include") + os.makedirs(bin_dir, exist_ok=True) + os.makedirs(lib_dir, exist_ok=True) os.makedirs(bin_dir, exist_ok=True) shutil.copytree(f"PCBuild\\{path}", bin_dir, dirs_exist_ok=True) + shutil.copytree(f"Lib", lib_dir, dirs_exist_ok=True) + shutil.copytree(f"Include", inc_dir, dirs_exist_ok=True) else: raise NotImplemented( - "Non-windows compilation of Python is not implemented yet" + "Non-Windows compilation of Python is not implemented yet" ) - def build_qt(self, options:dict): + def get_python_version(self) -> str: + path_to_python = os.path.join(self.install_dir, "bin", "python") + if sys.platform.startswith("win32"): + path_to_python += ".exe" + try: + result = subprocess.run([path_to_python, "--version"], capture_output=True, check=True) + _, _, version_number = result.stdout.decode("utf-8").strip().partition(" ") + components = version_number.split(".") + python_version = f"{components[0]}.{components[1]}" + return python_version + except subprocess.CalledProcessError as e: + print("ERROR: Failed to run LibPack's Python executable") + print(e.output.decode("utf-8")) + exit(1) + + def build_qt(self, options: dict): """Doesn't really "build" Qt, just copies the pre-compiled libraries from the configured path""" qt_dir = options["install-directory"] if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir,"metatypes")): - print("Skipping existing Qt") + if os.path.exists(os.path.join(self.install_dir, "metatypes")): + print("Not re-copying, instead just using existing Qt in the LibPack installation path") return if not os.path.exists(qt_dir): print(f"Error: specified Qt installation path does not exist ({qt_dir})") exit(1) shutil.copytree(qt_dir, self.install_dir, dirs_exist_ok=True) - def build_boost(self, options:dict=None): + def build_boost(self, _=None): """ Builds boost shared libraries and installs libraries and headers """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir,"include","boost")): - print("Skipping existing boost") + if os.path.exists(os.path.join(self.install_dir, "include", "boost")): + print("Not rebuilding boost, it is already in the LibPack") return - # Boost uses a custom build system and needs a config file \ No newline at end of file + # Boost uses a custom build system and needs a config file to find our Python + with open(os.path.join("tools", "build", "src", "user-config.jam"), "w", encoding="utf-8") as user_config: + exe = os.path.join(self.install_dir, "bin", "python") + if sys.platform.startswith("win32"): + exe += ".exe" + inc_dir = os.path.join(self.install_dir, "bin", "Include") + lib_dir = os.path.join(self.install_dir, "bin", "Lib") + python_version = self.get_python_version() + print(f"Building boost-python with Python {python_version}") + user_config.write(f'using python : {python_version} : "{exe}" : "{inc_dir}" : "{lib_dir}" ;\n') + try: + subprocess.run(["bootstrap.bat"], capture_output=True, check=True) + subprocess.run(["b2", f"variant={str(self.mode).lower()}"], check=True, capture_output=True) + shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) + shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), + dirs_exist_ok=True) + except subprocess.CalledProcessError as e: + print("Error: failed to build boost") + print(e.output.decode("utf-8")) + exit(e.returncode) diff --git a/config.json b/config.json index b3d27f5..5c1cf2c 100644 --- a/config.json +++ b/config.json @@ -2,11 +2,6 @@ "FreeCAD-version":"0.22", "LibPack-version":"3.0.0", "content": [ - { - "name":"python", - "git-repo":"https://github.com/python/cpython", - "git-ref":"v3.11.4" - }, { "name":"qt", "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" diff --git a/create_libpack.py b/create_libpack.py index 58a4294..b424f23 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -178,7 +178,7 @@ def decompress(name: str, filename: str): compiler = compile_all.Compiler( config, - compile_all.build_mode.RELEASE, + compile_all.BuildMode.RELEASE, bison_path=path_to_bison, skip_existing=args["skip_existing_build"], ) diff --git a/test_bootstrap.py b/test_bootstrap.py new file mode 100644 index 0000000..1b06674 --- /dev/null +++ b/test_bootstrap.py @@ -0,0 +1,72 @@ +#!/bin/python3 +import compile_all +# SPDX-License-Identifier: LGPL-2.1-or-later + +import os +import tempfile +import unittest + +import bootstrap + +""" Developer tests for the bootstrap module. """ + + +class TestBootstrap(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.original_dir = os.getcwd() + self.config = {"FreeCAD-version": "0.22", + "LibPack-version": "3.0.0", + "content": [ + {"name": "nonexistent"} + ]} + + def tearDown(self) -> None: + super().tearDown() + + def test_create_libpack_dir_no_conflict(self): + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + try: + dirname = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) + self.assertNotEqual(dirname.find("0.22"), -1) + self.assertNotEqual(dirname.find("3.0.0"), -1) + self.assertNotEqual(dirname.find("Release"), -1) + except: + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + raise + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + def test_create_libpack_dir_needs_backup(self): + with tempfile.TemporaryDirectory() as temp_dir: + os.chdir(temp_dir) + try: + first = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) + second = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) + self.assertTrue(os.path.exists(second)) + self.assertEqual(len(os.listdir()), 2) + except: + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + raise + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + def test_too_many_backups_exist(self): + with tempfile.TemporaryDirectory() as temp_dir: + try: + # Arrange + os.chdir(temp_dir) + config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} + for i in range(0, 27): + bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) + self.assertEqual(len(os.listdir()), i + 1) + # Act + with self.assertRaises(SystemExit): + bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) + except: + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + raise + os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + + +if __name__ == "__main__": + unittest.main() diff --git a/test_compile_all.py b/test_compile_all.py index 3038a2f..316c962 100644 --- a/test_compile_all.py +++ b/test_compile_all.py @@ -3,73 +3,52 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import os -import tempfile import unittest -from unittest.mock import MagicMock, mock_open, patch +from unittest.mock import MagicMock, patch import compile_all +""" Developer tests for the compile_all module. """ + class TestCompileAll(unittest.TestCase): def setUp(self) -> None: super().setUp() - config = {"FreeCAD-version": "0.22", + config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0", - "content":[ - {"name":"nonexistent"} - ]} - self.compiler = compile_all.Compiler(config, compile_all.build_mode.RELEASE, "bison_path") + "content": [ + {"name": "nonexistent"} + ]} + self.compiler = compile_all.Compiler(config, compile_all.BuildMode.RELEASE, "bison_path") self.original_dir = os.getcwd() def tearDown(self) -> None: os.chdir(self.original_dir) super().tearDown() - @patch("compile_all.Compiler.create_libpack_dir") @patch("os.chdir") @patch("compile_all.Compiler.build_nonexistent") - def test_compile_all_calls_build_function(self, nonexistent_mock:MagicMock, _1, _2): - config = {"content":[ - {"name":"nonexistent"} + def test_compile_all_calls_build_function(self, nonexistent_mock: MagicMock, _): + config = {"content": [ + {"name": "nonexistent"} ]} self.compiler.compile_all() nonexistent_mock.assert_called_once() - def test_create_libpack_dir_no_conflict(self): - with tempfile.TemporaryDirectory() as temp_dir: - os.chdir(temp_dir) - dirname = self.compiler.create_libpack_dir() - self.assertNotEqual(dirname.find("0.22"), -1) - self.assertNotEqual(dirname.find("3.0.0"), -1) - self.assertNotEqual(dirname.find("release"), -1) - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + @patch("subprocess.run") + def test_check_python_version(self, run_mock: MagicMock): + """ Checking the Python version stores the Major and Minor components (but not the Patch) """ - def test_create_libpack_dir_needs_backup(self): - with tempfile.TemporaryDirectory() as temp_dir: - os.chdir(temp_dir) - first = self.compiler.create_libpack_dir() - second = self.compiler.create_libpack_dir() - self.assertTrue(os.path.exists(second)) - self.assertEqual(len(os.listdir()), 2) - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + # Arrange + mock_result = MagicMock() + mock_result.stdout = b"Python 3.9.13" + run_mock.return_value = mock_result - def test_too_many_backups_exist(self): - with tempfile.TemporaryDirectory() as temp_dir: - try: - # Arrange - os.chdir(temp_dir) - config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} - for i in range(0,27): - self.compiler.create_libpack_dir() - self.assertEqual(len(os.listdir()), i+1) - # Act - with self.assertRaises(SystemExit): - self.compiler.create_libpack_dir() - except: - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - raise - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows + # Act + version = self.compiler.get_python_version() + # Assert + self.assertEqual(version, "3.9") if __name__ == "__main__": From 268104fe5c79dc4d2efb522b3bef630b335776b9 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 19 Aug 2023 21:42:28 -0500 Subject: [PATCH 05/27] Add patching tech and patch for Coin --- .gitignore | 4 + compile_all.py | 111 ++++++++++++++++++ config.json | 3 +- create_libpack.py | 1 + generate_patch.py | 37 ++++++ ...add-QOpenGLContext-to-QuarterWidgetP.patch | 5 + test_compile_all.py | 34 +++++- test_generate_patch.py | 50 ++++++++ 8 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 generate_patch.py create mode 100644 patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch create mode 100644 test_generate_patch.py diff --git a/.gitignore b/.gitignore index d1805d9..1af08ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ __pycache__ .vscode/ working +LibPack-* +.venv +venv +.idea \ No newline at end of file diff --git a/compile_all.py b/compile_all.py index 2deb25b..0715688 100644 --- a/compile_all.py +++ b/compile_all.py @@ -5,9 +5,12 @@ # Every package has its own compilation and installation idiosyncrasies, so we have to use a custom # build script for each one. +from diff_match_patch import diff_match_patch from enum import Enum import os +import pathlib import platform +import re import shutil import subprocess import sys @@ -26,6 +29,56 @@ def __str__(self) -> str: return "Unknown" +def patch_single_file(filename, patch_data) -> None: + with open(filename, "r", encoding="utf-8") as f: + original_data = f.read() + dmp = diff_match_patch() + patches = dmp.patch_fromText(patch_data) + new_text, applied = dmp.patch_apply(patches, original_data) + if not all(applied): + print(f"ERROR: Failed to apply some patches to {filename}") + # TODO: Someday actually print out what patches failed? + exit(1) + with open (filename, "w", encoding="utf-8") as f: + f.write(new_text) + + +def split_patch_data(patch_data: str) -> list[dict[str, str]]: + filename_regex = re.compile("@@@ ([^@]*) @@@\n") + split_data = re.split(filename_regex, patch_data) + result = [] + for index, entry in enumerate(split_data): + if index == 0: + if entry != "": + print("ERROR: Bad patch file, must start with @@@ filename @@@") + exit(1) + continue + if index % 2 == 1: + result.append({"file": entry}) + else: + result[-1]["data"] = entry + return result + + +def apply_patch(patch_file_path: str) -> None: + """ Apply a patch that was generated by the generate_patch.py script """ + # Path is relative to *this* file, not our working directory + absolute_path = os.path.join(pathlib.Path(__file__).parent.absolute(), patch_file_path) + with open(absolute_path, "r", encoding="utf-8") as f: + patch_data = f.read() + patches = split_patch_data(patch_data) + for patch in patches: + patch_single_file(patch["file"], patch["data"]) + + +def patch_files(patches: list[str]) -> None: + """ Given a list of patches, apply them sequentially in the current working directory. The patches themselves are + expected to be given as paths relative to **this** Python script file""" + for patch in patches: + start = len("patches/") + print(f"Applying patch {patch[start:]}") + apply_patch(patch) + class Compiler: def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.config = config @@ -40,11 +93,37 @@ def __init__(self, config, mode, bison_path, skip_existing: bool = False): ) self.install_dir = os.path.join(os.getcwd(), libpack_dir) + def get_cmake_options(self) -> list[str]: + """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply + to all builds, but none conflict. """ + return [ + f"-DBUILD_TESTS=No", + f"-DBUILD_EXAMPLES=No", + f"-DBUILD_DOCS=No", + f"-DCMAKE_INSTALL_PATH={self.install_dir}", + f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", + f"-DPython_ROOT_DIR={self.install_dir}/bin", + f"-DBOOST_ROOT={self.install_dir}", + f"-DBOOST_INCLUDE_DIR={self.install_dir}/include", + f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", + f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", + f"-DSWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", + f"-DSWIG_DIR={self.install_dir}/bin/swig/Lib", + f"-DZLIB_INCLUDE_DIR={self.install_dir}/include", + f"-DZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith("win32") else "a", + f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", + f"-DZLIB_DIR={self.install_dir}/lib/cmake/", + f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", + f"-DCMAKE_CXX_FLAGS=-I{self.install_dir}/include" + ] + def compile_all(self): for item in self.config["content"]: # All build methods are named using "build_XXX" where XXX is the name of the package in the config file print(f"Building {item['name']} in {self.mode} mode") os.chdir(item["name"]) + if "patches" in item: + patch_files(item["patches"]) build_function_name = "build_" + item["name"] build_function = getattr(self, build_function_name) build_function(item) @@ -145,3 +224,35 @@ def build_boost(self, _=None): print("Error: failed to build boost") print(e.output.decode("utf-8")) exit(e.returncode) + + def _build_standard_cmake(self): + build_dir = "build-" + str(self.mode).lower() + if not os.path.exists(build_dir): + os.mkdir(build_dir) + os.chdir(build_dir) + + cmake_setup_options = ["cmake"] + cmake_setup_options.extend(self.get_cmake_options()) + cmake_setup_options.append("..") + cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] + cmake_install_options = ["cmake", "--install", "."] + try: + subprocess.run(cmake_setup_options, check=True, capture_output=True) + subprocess.run(cmake_build_options, check=True, capture_output=True) + subprocess.run(cmake_install_options, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Build failed!") + print(e.output.decode("utf-8")) + exit(e.returncode) + + def build_coin(self, _=None): + """ Builds and installs Coin using standard CMake settings """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "share", "Coin")): + print("Not rebuilding Coin, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_quarter(self, options=None): + """ Builds and installs Quarter using standard CMake settings """ + self._build_standard_cmake() diff --git a/config.json b/config.json index 5c1cf2c..62eeee9 100644 --- a/config.json +++ b/config.json @@ -19,7 +19,8 @@ { "name":"quarter", "git-repo":"https://github.com/coin3d/quarter", - "git-ref":"master" + "git-ref":"master", + "patches":["patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch"] }, { "name":"pcre2", diff --git a/create_libpack.py b/create_libpack.py index b424f23..9db69cd 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -10,6 +10,7 @@ # * 7z (see https://www.7-zip.org) # * Some version of Python that can run this file # * The "requests" Python package (e.g. 'pip install requests') +# * The "diff-match-patch" Python package (e.g. 'pip install diff-match-patch') # * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF # * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) diff --git a/generate_patch.py b/generate_patch.py new file mode 100644 index 0000000..8d361ff --- /dev/null +++ b/generate_patch.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +import diff_match_patch +import sys + + +def print_usage(): + print("Generate a patchfile that can be used with the create_libpack.py script to patch source files") + print("Usage: python generate_patch.py original_file corrected_file output_patch_file") + + +def parse_args(): + if len(sys.argv) != 4: + print_usage() + exit(1) + + +def generate_patch(old, new) -> str: + dmp = diff_match_patch.diff_match_patch() + patches = dmp.patch_make(old, new) + return dmp.patch_toText(patches) + + +def run(old_file, new_file, output_file): + with open(old_file, "r", encoding="utf-8") as f: + old = f.read() + with open(new_file, "r", encoding="utf-8") as f: + new = f.read() + patch = generate_patch(old, new) + with open(output_file, "w", encoding="utf-8") as f: + f.write(f"@@@ {old_file} @@@\n") + f.write(patch) + + +if __name__ == "__main__": + parse_args() + run(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch b/patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch new file mode 100644 index 0000000..74c104f --- /dev/null +++ b/patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch @@ -0,0 +1,5 @@ +@@@ src\Quarter\QuarterWidgetP.cpp @@@ +@@ -2303,24 +2303,51 @@ + glue/gl.h%3E%0A%0A ++#include %3CQOpenGLContext%3E%0A%0A + #include %22Na diff --git a/test_compile_all.py b/test_compile_all.py index 316c962..efb2066 100644 --- a/test_compile_all.py +++ b/test_compile_all.py @@ -4,7 +4,7 @@ import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, mock_open import compile_all @@ -36,7 +36,7 @@ def test_compile_all_calls_build_function(self, nonexistent_mock: MagicMock, _): nonexistent_mock.assert_called_once() @patch("subprocess.run") - def test_check_python_version(self, run_mock: MagicMock): + def test_get_python_version(self, run_mock: MagicMock): """ Checking the Python version stores the Major and Minor components (but not the Patch) """ # Arrange @@ -50,6 +50,36 @@ def test_check_python_version(self, run_mock: MagicMock): # Assert self.assertEqual(version, "3.9") + def test_split_patch_data_finds_single_file(self): + filename = "filename" + patch_data = "@@ -1,3 +1,1 @@ -The +A" + data = f"@@@ {filename} @@@\n{patch_data}" + patches = compile_all.split_patch_data(data) + self.assertEqual(1, len(patches)) + self.assertEqual(patches[0]["file"], filename) + + def test_split_patch_data_finds_multiple_files(self): + filename = "filename" + patch_data = "@@ -1,3 +1,1 @@\n-The +A" + expected_number = 5 + data = "" + for i in range(expected_number): + data += f"@@@ {filename}{i} @@@\n{patch_data}" + patches = compile_all.split_patch_data(data) + self.assertEqual(expected_number, len(patches)) + + +class TestPatchSingleFile(unittest.TestCase): + + @patch("builtins.open", mock_open(read_data="The End.")) + def test_integration_patch_single_file(self): + mo = mock_open(read_data="The End.") + with patch("builtins.open", mo): + compile_all.patch_single_file("filename", "@@ -1,7 +1,5 @@\n-The\n+A\n End\n") + mo.assert_any_call("filename", "w", encoding="utf-8") + handle = mo() + handle.write.assert_called_once_with("A End.") + if __name__ == "__main__": unittest.main() diff --git a/test_generate_patch.py b/test_generate_patch.py new file mode 100644 index 0000000..6ae0272 --- /dev/null +++ b/test_generate_patch.py @@ -0,0 +1,50 @@ +#!/bin/python3 + +# SPDX-License-Identifier: LGPL-2.1-or-later + +import diff_match_patch +import unittest +from unittest.mock import MagicMock, patch, mock_open + +import generate_patch + +""" Developer tests for the generate_patch module. """ + +class TestGeneratePatch(unittest.TestCase): + + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + @patch("sys.argv", ["exe", "old", "new", "patch"]) + def test_command_line_options_count_is_correct(self): + generate_patch.parse_args() + + @patch("sys.argv", ["exe"]) + def test_command_line_args_missing_is_error(self): + with self.assertRaises(SystemExit): + generate_patch.parse_args() + + def test_patch_generated(self): + # Arrange + old = "Line1\nLine2\nLine4\n" + new = "Line1\nLine2\nLine3\n" + dmp = diff_match_patch.diff_match_patch() + difference = dmp.patch_toText(dmp.patch_make(old, new)) + + # Act + result = generate_patch.generate_patch(old, new) + + # Assert + self.assertEqual(result, difference) + + def test_run_loads_all_files(self): + with patch("builtins.open", mock_open()) as open_mock: + generate_patch.run("old", "new", "patch") + expected_calls = [unittest.mock.call("old","r",encoding="utf-8"), + unittest.mock.call("new","r",encoding="utf-8"), + unittest.mock.call("patch","w",encoding="utf-8")] + for call in expected_calls: + self.assertIn(call, open_mock.mock_calls) From 4807250b56700e8e27849b575724904f9c9b5cee Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 20 Aug 2023 00:24:11 -0500 Subject: [PATCH 06/27] Add compilation of PCRE2, SWIG, and pivy --- compile_all.py | 59 ++++++++++++++++++++++++++++++++++++++++++++--- config.json | 20 ++++++++-------- create_libpack.py | 2 ++ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/compile_all.py b/compile_all.py index 0715688..f983432 100644 --- a/compile_all.py +++ b/compile_all.py @@ -92,6 +92,7 @@ def __init__(self, config, mode, bison_path, skip_existing: bool = False): str(mode), ) self.install_dir = os.path.join(os.getcwd(), libpack_dir) + self.init_script = None def get_cmake_options(self) -> list[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply @@ -100,6 +101,7 @@ def get_cmake_options(self) -> list[str]: f"-DBUILD_TESTS=No", f"-DBUILD_EXAMPLES=No", f"-DBUILD_DOCS=No", + f"-DBUILD_SHARED=Yes", f"-DCMAKE_INSTALL_PATH={self.install_dir}", f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", f"-DPython_ROOT_DIR={self.install_dir}/bin", @@ -108,13 +110,14 @@ def get_cmake_options(self) -> list[str]: f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", f"-DSWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", - f"-DSWIG_DIR={self.install_dir}/bin/swig/Lib", f"-DZLIB_INCLUDE_DIR={self.install_dir}/include", f"-DZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith("win32") else "a", f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", f"-DZLIB_DIR={self.install_dir}/lib/cmake/", f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", - f"-DCMAKE_CXX_FLAGS=-I{self.install_dir}/include" + f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", + f"-DCMAKE_CXX_FLAGS=-I{self.install_dir}/include", + f"-DBISON_EXECUTABLE={self.bison_path}" ] def compile_all(self): @@ -253,6 +256,56 @@ def build_coin(self, _=None): return self._build_standard_cmake() - def build_quarter(self, options=None): + def build_quarter(self, _=None): """ Builds and installs Quarter using standard CMake settings """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "Quarter")): + print("Not rebuilding Coin, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_zlib(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "zlib.h")): + print("Not rebuilding zlib, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_bzip2(self, _= None): + """ The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): + print("Not rebuilding zlib, it is already in the LibPack") + return + if sys.platform.startswith("win32"): + args = [self.init_script, "&", "nmake","/f", "makefile.msc"] + try: + subprocess.run(args, check=True, capture_output=True) + shutil.copyfile("libbz2.lib",os.path.join(self.install_dir, "lib", "libbz2.lib")) + shutil.copyfile("bzlib.h",os.path.join(self.install_dir, "include", "bzlib.h")) + shutil.copyfile("bzlib_private.h",os.path.join(self.install_dir, "include", "bzlib_private.h")) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to build bzip2 using nmake") + print(e.output.decode("utf-8")) + exit(1) + else: + raise NotImplemented( + "Non-Windows compilation of bzip2 is not implemented yet" + ) + + def build_pcre2(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "pcre2.h")): + print("Not rebuilding pcre2, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_swig(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "bin", "swig") + ".exe" if sys.platform.startswith("win32") else ""): + print("Not rebuilding SWIG, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_pivy(self, _=None): self._build_standard_cmake() diff --git a/config.json b/config.json index 62eeee9..ca2570a 100644 --- a/config.json +++ b/config.json @@ -22,6 +22,16 @@ "git-ref":"master", "patches":["patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch"] }, + { + "name":"zlib", + "git-repo":"https://github.com/madler/zlib", + "git-ref":"v1.3" + }, + { + "name":"bzip2", + "git-repo":"https://gitlab.com/bzip2/bzip2.git", + "git-ref":"bzip2-1.0.8" + }, { "name":"pcre2", "git-repo":"https://github.com/PCRE2Project/pcre2", @@ -55,21 +65,11 @@ "git-repo":"https://github.com/harfbuzz/harfbuzz", "git-ref":"8.1.1" }, - { - "name":"zlib", - "git-repo":"https://github.com/intel/zlib", - "git-ref":"v1.2.13_jtk" - }, { "name":"libpng", "git-repo":"https://github.com/glennrp/libpng", "git-ref":"v1.6.40" }, - { - "name":"bzip2", - "git-repo":"https://gitlab.com/bzip2/bzip2.git", - "git-ref":"bzip2-1.0.8" - }, { "name":"freetype", "git-repo":"https://gitlab.freedesktop.org/freetype/freetype/", diff --git a/create_libpack.py b/create_libpack.py index 9db69cd..3eeea1d 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -27,6 +27,7 @@ path_to_7zip = "C:\\Program Files\\7-Zip\\7z.exe" path_to_bison = "C:\\Program Files\\win_flex_bison\\win_bison.exe" +devel_init_script = "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" def remove_readonly(func, path, _) -> None: @@ -183,6 +184,7 @@ def decompress(name: str, filename: str): bison_path=path_to_bison, skip_existing=args["skip_existing_build"], ) + compiler.init_script = devel_init_script compiler.compile_all() From 980940b27d824a72ec7b04789743024594c45aac Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 20 Aug 2023 12:01:56 -0500 Subject: [PATCH 07/27] Begin adding VTK (incomplete) --- compile_all.py | 67 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/compile_all.py b/compile_all.py index f983432..378410d 100644 --- a/compile_all.py +++ b/compile_all.py @@ -39,7 +39,7 @@ def patch_single_file(filename, patch_data) -> None: print(f"ERROR: Failed to apply some patches to {filename}") # TODO: Someday actually print out what patches failed? exit(1) - with open (filename, "w", encoding="utf-8") as f: + with open(filename, "w", encoding="utf-8") as f: f.write(new_text) @@ -79,6 +79,7 @@ def patch_files(patches: list[str]) -> None: print(f"Applying patch {patch[start:]}") apply_patch(patch) + class Compiler: def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.config = config @@ -97,7 +98,7 @@ def __init__(self, config, mode, bison_path, skip_existing: bool = False): def get_cmake_options(self) -> list[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ - return [ + base = [ f"-DBUILD_TESTS=No", f"-DBUILD_EXAMPLES=No", f"-DBUILD_DOCS=No", @@ -116,9 +117,15 @@ def get_cmake_options(self) -> list[str]: f"-DZLIB_DIR={self.install_dir}/lib/cmake/", f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", - f"-DCMAKE_CXX_FLAGS=-I{self.install_dir}/include", f"-DBISON_EXECUTABLE={self.bison_path}" ] + CXX_FLAGS = "" + if sys.platform.startswith("win32"): + CXX_FLAGS = "/I{self.install_dir}/include /EHsc" + else: + CXX_FLAGS="-I{self.install_dir}/include" + base.append(f"-DCMAKE_CXX_FLAGS={CXX_FLAGS}") + return base def compile_all(self): for item in self.config["content"]: @@ -228,14 +235,15 @@ def build_boost(self, _=None): print(e.output.decode("utf-8")) exit(e.returncode) - def _build_standard_cmake(self): + def _build_standard_cmake(self, extra_cxx_flags=""): build_dir = "build-" + str(self.mode).lower() if not os.path.exists(build_dir): os.mkdir(build_dir) os.chdir(build_dir) cmake_setup_options = ["cmake"] - cmake_setup_options.extend(self.get_cmake_options()) + standard_options = self.get_cmake_options() + cmake_setup_options.extend(standard_options) cmake_setup_options.append("..") cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] cmake_install_options = ["cmake", "--install", "."] @@ -271,19 +279,19 @@ def build_zlib(self, _=None): return self._build_standard_cmake() - def build_bzip2(self, _= None): + def build_bzip2(self, _=None): """ The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): print("Not rebuilding zlib, it is already in the LibPack") return if sys.platform.startswith("win32"): - args = [self.init_script, "&", "nmake","/f", "makefile.msc"] + args = [self.init_script, "&", "nmake", "/f", "makefile.msc"] try: subprocess.run(args, check=True, capture_output=True) - shutil.copyfile("libbz2.lib",os.path.join(self.install_dir, "lib", "libbz2.lib")) - shutil.copyfile("bzlib.h",os.path.join(self.install_dir, "include", "bzlib.h")) - shutil.copyfile("bzlib_private.h",os.path.join(self.install_dir, "include", "bzlib_private.h")) + shutil.copyfile("libbz2.lib", os.path.join(self.install_dir, "lib", "libbz2.lib")) + shutil.copyfile("bzlib.h", os.path.join(self.install_dir, "include", "bzlib.h")) + shutil.copyfile("bzlib_private.h", os.path.join(self.install_dir, "include", "bzlib_private.h")) except subprocess.CalledProcessError as e: print("ERROR: Failed to build bzip2 using nmake") print(e.output.decode("utf-8")) @@ -302,10 +310,47 @@ def build_pcre2(self, _=None): def build_swig(self, _=None): if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "bin", "swig") + ".exe" if sys.platform.startswith("win32") else ""): + if os.path.exists( + os.path.join(self.install_dir, "bin", "swig") + ".exe" if sys.platform.startswith("win32") else ""): print("Not rebuilding SWIG, it is already in the LibPack") return self._build_standard_cmake() def build_pivy(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy")): + print("Not rebuilding pivy, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_libclang(self, _=None): + """ libclang is provided as a platform-specific download by Qt. """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "clang")): + print("Not copying libclang, it is already in the LibPack") + return + shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True) + + def build_pyside(self, options=None): + """ As of Qt6, pyside is installed using pip """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")): + print("Not rebuilding PySide6, it is already in the LibPack") + return + if "pip-install" not in options: + print("ERROR: No pip-install provided in configuration of pyside, so version cannot be determined") + exit(1) + path_to_python = os.path.join(self.install_dir, "bin", "python") + if sys.platform.startswith("win32"): + path_to_python += ".exe" + try: + subprocess.run([path_to_python, "-m", "pip", "install", options['pip-install']], check=True, + capture_output=True) + except subprocess.CalledProcessError as e: + print(f"ERROR: Failed to pip install {options['pip-install']}") + print(e.output.decode("utf-8")) + exit(1) + + def build_vtk(self, _=None): self._build_standard_cmake() + From 20e823914a97031cba0963432987c18c836e6414 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 20 Aug 2023 17:56:17 -0500 Subject: [PATCH 08/27] Cleanup of output and CXX_FLAGS --- bootstrap.py | 8 +++++++- compile_all.py | 32 ++++++++++++++++++-------------- create_libpack.py | 6 ++---- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index f4ba4be..cfe21d5 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -32,6 +32,8 @@ def usage(): print("Usage: python bootstrap.py [config_file] [release|debug]") print() print('Result: A new working/LibPack-XX-YY-MM directory has been created') + +def print_next_step(): print('Next step: install Python into working/LibPack-XX-YY-MM/bin, then run:') print(' .\\working\\LibPack-XX-YY-MM\\bin\\python create_libpack.py') print('(where XX, YY, and MM change according to the config and inputs)') @@ -58,6 +60,9 @@ def create_libpack_dir(config: dict, mode: BuildMode) -> str: backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) os.rename(dirname, backup_name) + if not os.path.exists(dirname): + os.mkdir(dirname) + dirname = os.pawth.join(dirname, "bin") if not os.path.exists(dirname): os.mkdir(dirname) return dirname @@ -67,7 +72,8 @@ def create_libpack_dir(config: dict, mode: BuildMode) -> str: args = parse_args() with open(args["config_file"], "r", encoding="utf-8") as f: config_data = f.read() - config_dict = json.load(config_data) + config_dict = json.loads(config_data) os.makedirs("working", exist_ok=True) os.chdir("working") create_libpack_dir(config_dict, args["mode"]) + print_next_step() \ No newline at end of file diff --git a/compile_all.py b/compile_all.py index 378410d..b1e99bc 100644 --- a/compile_all.py +++ b/compile_all.py @@ -103,11 +103,14 @@ def get_cmake_options(self) -> list[str]: f"-DBUILD_EXAMPLES=No", f"-DBUILD_DOCS=No", f"-DBUILD_SHARED=Yes", + f"-DBUILD_SHARED_LIB=Yes", + f"-DBUILD_SHARED_LIBS=Yes", f"-DCMAKE_INSTALL_PATH={self.install_dir}", f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", f"-DPython_ROOT_DIR={self.install_dir}/bin", f"-DBOOST_ROOT={self.install_dir}", - f"-DBOOST_INCLUDE_DIR={self.install_dir}/include", + f"-DBoost_INCLUDE_DIR={self.install_dir}/include", + f"-DBoost_INCLUDE_DIRS={self.install_dir}/include", f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", f"-DSWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", @@ -121,9 +124,9 @@ def get_cmake_options(self) -> list[str]: ] CXX_FLAGS = "" if sys.platform.startswith("win32"): - CXX_FLAGS = "/I{self.install_dir}/include /EHsc" + CXX_FLAGS = f"/I{self.install_dir}/include /EHsc" else: - CXX_FLAGS="-I{self.install_dir}/include" + CXX_FLAGS= f"-I{self.install_dir}/include" base.append(f"-DCMAKE_CXX_FLAGS={CXX_FLAGS}") return base @@ -201,18 +204,19 @@ def build_qt(self, options: dict): qt_dir = options["install-directory"] if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "metatypes")): - print("Not re-copying, instead just using existing Qt in the LibPack installation path") + print(" Not re-copying, instead just using existing Qt in the LibPack installation path") return if not os.path.exists(qt_dir): print(f"Error: specified Qt installation path does not exist ({qt_dir})") exit(1) + print(" (Note that Qt isn't really 'built,' it is just copied from a local installation)") shutil.copytree(qt_dir, self.install_dir, dirs_exist_ok=True) def build_boost(self, _=None): """ Builds boost shared libraries and installs libraries and headers """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "boost")): - print("Not rebuilding boost, it is already in the LibPack") + print(" Not rebuilding boost, it is already in the LibPack") return # Boost uses a custom build system and needs a config file to find our Python with open(os.path.join("tools", "build", "src", "user-config.jam"), "w", encoding="utf-8") as user_config: @@ -260,7 +264,7 @@ def build_coin(self, _=None): """ Builds and installs Coin using standard CMake settings """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "share", "Coin")): - print("Not rebuilding Coin, it is already in the LibPack") + print(" Not rebuilding Coin, it is already in the LibPack") return self._build_standard_cmake() @@ -268,14 +272,14 @@ def build_quarter(self, _=None): """ Builds and installs Quarter using standard CMake settings """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "Quarter")): - print("Not rebuilding Coin, it is already in the LibPack") + print(" Not rebuilding Quarter, it is already in the LibPack") return self._build_standard_cmake() def build_zlib(self, _=None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "zlib.h")): - print("Not rebuilding zlib, it is already in the LibPack") + print(" Not rebuilding zlib, it is already in the LibPack") return self._build_standard_cmake() @@ -283,7 +287,7 @@ def build_bzip2(self, _=None): """ The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): - print("Not rebuilding zlib, it is already in the LibPack") + print(" Not rebuilding bzip2, it is already in the LibPack") return if sys.platform.startswith("win32"): args = [self.init_script, "&", "nmake", "/f", "makefile.msc"] @@ -304,7 +308,7 @@ def build_bzip2(self, _=None): def build_pcre2(self, _=None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "pcre2.h")): - print("Not rebuilding pcre2, it is already in the LibPack") + print(" Not rebuilding pcre2, it is already in the LibPack") return self._build_standard_cmake() @@ -312,14 +316,14 @@ def build_swig(self, _=None): if self.skip_existing: if os.path.exists( os.path.join(self.install_dir, "bin", "swig") + ".exe" if sys.platform.startswith("win32") else ""): - print("Not rebuilding SWIG, it is already in the LibPack") + print(" Not rebuilding SWIG, it is already in the LibPack") return self._build_standard_cmake() def build_pivy(self, _=None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy")): - print("Not rebuilding pivy, it is already in the LibPack") + print(" Not rebuilding pivy, it is already in the LibPack") return self._build_standard_cmake() @@ -327,7 +331,7 @@ def build_libclang(self, _=None): """ libclang is provided as a platform-specific download by Qt. """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "clang")): - print("Not copying libclang, it is already in the LibPack") + print(" Not copying libclang, it is already in the LibPack") return shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True) @@ -335,7 +339,7 @@ def build_pyside(self, options=None): """ As of Qt6, pyside is installed using pip """ if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")): - print("Not rebuilding PySide6, it is already in the LibPack") + print(" Not rebuilding PySide6, it is already in the LibPack") return if "pip-install" not in options: print("ERROR: No pip-install provided in configuration of pyside, so version cannot be determined") diff --git a/create_libpack.py b/create_libpack.py index 3eeea1d..92b1f7e 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -170,10 +170,8 @@ def decompress(name: str, filename: str): path_to_7zip = args["7zip"] path_to_bison = args["bison"] - if not args["skip_existing_clone"]: - delete_existing(args["working"], silent=args["silent"]) - - os.makedirs(args["working"], exist_ok=True) + if not os.path.exists(os.path.join("working","bin")): + print("ERROR: ") os.chdir(args["working"]) fetch_remote_data(config, args["skip_existing_clone"]) From 7101c015667eb33e4b5fa8a1c8b9cda68d3c3b77 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 20 Aug 2023 20:35:09 -0500 Subject: [PATCH 09/27] Add libpng, harfbuzz, freetype, tcl, and tk --- compile_all.py | 91 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/compile_all.py b/compile_all.py index b1e99bc..1a700fa 100644 --- a/compile_all.py +++ b/compile_all.py @@ -99,28 +99,30 @@ def get_cmake_options(self) -> list[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ base = [ - f"-DBUILD_TESTS=No", - f"-DBUILD_EXAMPLES=No", + f"-DBISON_EXECUTABLE={self.bison_path}", + f"-DBOOST_ROOT={self.install_dir}", f"-DBUILD_DOCS=No", + f"-DBUILD_EXAMPLES=No", f"-DBUILD_SHARED=Yes", f"-DBUILD_SHARED_LIB=Yes", f"-DBUILD_SHARED_LIBS=Yes", + f"-DBUILD_TESTS=No", + f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", + f"-DBoost_INCLUDE_DIR={self.install_dir}/include", + f"-DBoost_INCLUDE_DIRS={self.install_dir}/include", f"-DCMAKE_INSTALL_PATH={self.install_dir}", f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", + f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", + f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", + f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", f"-DPython_ROOT_DIR={self.install_dir}/bin", - f"-DBOOST_ROOT={self.install_dir}", - f"-DBoost_INCLUDE_DIR={self.install_dir}/include", - f"-DBoost_INCLUDE_DIRS={self.install_dir}/include", f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", - f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", f"-DSWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", + f"-DVTK_MODULE_ENABLE_VTK_IOIOSS=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-DVTK_MODULE_ENABLE_VTK_ioss=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-DZLIB_DIR={self.install_dir}/lib/cmake/", f"-DZLIB_INCLUDE_DIR={self.install_dir}/include", f"-DZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith("win32") else "a", - f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", - f"-DZLIB_DIR={self.install_dir}/lib/cmake/", - f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", - f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", - f"-DBISON_EXECUTABLE={self.bison_path}" ] CXX_FLAGS = "" if sys.platform.startswith("win32"): @@ -356,5 +358,72 @@ def build_pyside(self, options=None): exit(1) def build_vtk(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "share", "licenses", "VTK")): + print(" Not rebuilding VTK, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_harfbuzz(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "harfbuzz")): + print(" Not rebuilding harfbuzz, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_libpng(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "lib", "libpng")): + print(" Not rebuilding libpng, it is already in the LibPack") + return self._build_standard_cmake() + def build_freetype(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "freetype2")): + print(" Not rebuilding freetype, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_tcl(self, _=None): + """ tcl does not use cMake """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): + pass + if sys.platform.startswith("win32"): + try: + os.chdir("win") + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", f"INSTALLDIR={self.install_dir}"] + subprocess.run(args, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to build tcl using nmake") + print(e.output.decode("utf-8")) + exit(1) + else: + raise NotImplemented( + "Non-Windows compilation of tcl is not implemented yet" + ) + + def build_tk(self, _=None): + """ tk does not use cMake """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): + pass + if sys.platform.startswith("win32"): + try: + os.chdir("win") + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", f"INSTALLDIR={self.install_dir}"] + subprocess.run(args, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to build tk using nmake") + print(e.output.decode("utf-8")) + exit(1) + else: + raise NotImplemented( + "Non-Windows compilation of tk is not implemented yet" + ) + From 36fadae0a3ef579abf6867882110b260871cbf14 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 20 Aug 2023 23:52:14 -0500 Subject: [PATCH 10/27] Continued progress with compilation scripting --- compile_all.py | 111 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/compile_all.py b/compile_all.py index 1a700fa..568fbcb 100644 --- a/compile_all.py +++ b/compile_all.py @@ -106,7 +106,9 @@ def get_cmake_options(self) -> list[str]: f"-DBUILD_SHARED=Yes", f"-DBUILD_SHARED_LIB=Yes", f"-DBUILD_SHARED_LIBS=Yes", + f"-DBUILD_TEST=No", f"-DBUILD_TESTS=No", + f"-DBUILD_TESTING=No", f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", f"-DBoost_INCLUDE_DIR={self.install_dir}/include", f"-DBoost_INCLUDE_DIRS={self.install_dir}/include", @@ -114,6 +116,11 @@ def get_cmake_options(self) -> list[str]: f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", + f"-DHDF5_DIR={self.install_dir}/share/cmake/", + f"-DHDF5_LIBRARY_DEBUG=LIBPACK/lib/hdf5.lib", + f"-DHDF5_LIBRARY_RELEASE=LIBPACK/lib/hdf5.lib", + f"-DHDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + ".exe" if sys.platform.startswith("win32") else "", + f"-DINSTALL_DIR={self.install_dir}", f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", f"-DPython_ROOT_DIR={self.install_dir}/bin", f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", @@ -124,9 +131,9 @@ def get_cmake_options(self) -> list[str]: f"-DZLIB_INCLUDE_DIR={self.install_dir}/include", f"-DZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith("win32") else "a", ] - CXX_FLAGS = "" if sys.platform.startswith("win32"): - CXX_FLAGS = f"/I{self.install_dir}/include /EHsc" + inc_path = self.install_dir.replace('\\', '/') + CXX_FLAGS = f"/I{inc_path}/include /EHsc /DWIN32" else: CXX_FLAGS= f"-I{self.install_dir}/include" base.append(f"-DCMAKE_CXX_FLAGS={CXX_FLAGS}") @@ -241,7 +248,7 @@ def build_boost(self, _=None): print(e.output.decode("utf-8")) exit(e.returncode) - def _build_standard_cmake(self, extra_cxx_flags=""): + def _build_standard_cmake(self, extra_args: list[str] = None): build_dir = "build-" + str(self.mode).lower() if not os.path.exists(build_dir): os.mkdir(build_dir) @@ -250,6 +257,8 @@ def _build_standard_cmake(self, extra_cxx_flags=""): cmake_setup_options = ["cmake"] standard_options = self.get_cmake_options() cmake_setup_options.extend(standard_options) + if extra_args: + cmake_setup_options.extend(extra_args) cmake_setup_options.append("..") cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] cmake_install_options = ["cmake", "--install", "."] @@ -388,14 +397,21 @@ def build_freetype(self, _=None): def build_tcl(self, _=None): """ tcl does not use cMake """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): - pass + if os.path.exists(os.path.join(self.install_dir, "include", "tcl.h")): + print(" Not rebuilding tcl, it is already in the LibPack") + return if sys.platform.startswith("win32"): try: os.chdir("win") args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", f"INSTALLDIR={self.install_dir}"] + args = [self.init_script, + "&", + "nmake", + "/f", + "makefile.vc", + "install", + f"INSTALLDIR={self.install_dir}"] subprocess.run(args, check=True, capture_output=True) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tcl using nmake") @@ -409,8 +425,9 @@ def build_tcl(self, _=None): def build_tk(self, _=None): """ tk does not use cMake """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): - pass + if os.path.exists(os.path.join(self.install_dir, "include", "tk.h")): + print(" Not rebuilding tk, it is already in the LibPack") + return if sys.platform.startswith("win32"): try: os.chdir("win") @@ -427,3 +444,81 @@ def build_tk(self, _=None): "Non-Windows compilation of tk is not implemented yet" ) + def build_opencascade(self, _=None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): + print(" Not rebuilding OpenCASCADE, it is already in the LibPack") + return + extra_args = [f"-D3RDPARTY_DIR={self.install_dir}", "-DUSE_VTK=ON", + f"-D3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2"] + if self.mode == BuildMode.DEBUG: + extra_args.append("-DBUILD_SHARED_LIBRARY_NAME_POSTFIX=d") + self._build_standard_cmake(extra_args=extra_args) + + def build_netgen(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "share", "netgen")): + print(" Not rebuilding netgen, it is already in the LibPack") + return + extra_args = ["-D USE_SUPERBUILD=OFF", + "-D USE_GUI=OFF", + f"-D USE_INTERNAL_TCL=OFF", + f"-D TCL_DIR={self.install_dir}", + f"-D TK_DIR={self.install_dir}"] + self._build_standard_cmake(extra_args=extra_args) + + def build_hdf5(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "hdf5.h")): + print(" Not rebuilding hdf5, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_medfile(self, _: None): + print(" *** Salome MED File source is not currently available -- skipping in this build ***") + return + #extra_args = ["-DMEDFILE_USE_UNICODE=On"] + #self._build_standard_cmake(extra_args) + + def build_gmsh(self, _: None): + if self.skip_existing: + if os.path.exists( + os.path.join(self.install_dir, + "bin", + "gmsh" + ".exe" if sys.platform.startswith("win32") else "")): + print(" Not rebuilding gmsh, it is already in the LibPack") + return + extra_args = [] + if sys.platform.startswith("win32"): + extra_args = [f"-D CMAKE_LIBRARY_PATH={self.install_dir}/win64/vc14/lib", # TODO - Remove hardcoding + "-D ENABLE_OPENMP=No"] # Build fails if OpenMP is enabled + self._build_standard_cmake(extra_args) + + def build_pycxx(self, _: None): + """ PyCXX does not use a cMake-based build system """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "CXX")): + print(" Not rebuilding PyCXX, it is already in the LibPack") + return + path_to_python = os.path.join(self.install_dir, "bin", "python") + if sys.platform.startswith("win32"): + path_to_python += ".exe" + args = [path_to_python, "setup.py", "install"] + try: + subprocess.run(args, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to build PyCXX using its custom build script") + print(e.output.decode("utf-8")) + exit(1) + + def build_icu(self, _: None): + """ ICU does not use cMake, but has projects for various OSes """ + + if sys.platform.startswith("win32"): + args = ["msbuild", f"/p:Configuration={self.mode}", "/t:Build", "allinone.sln"] + subprocess.run(args, check=True, capture_output=True) + else: + raise NotImplemented( + "Non-Windows compilation of ICU is not implemented yet" + ) + From c47d249d3e946a1155ccaf9c86750e503ce06629 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Mon, 21 Aug 2023 22:11:38 -0500 Subject: [PATCH 11/27] Fix tcl/tk and OpenCASCADE, and add ICU --- compile_all.py | 169 ++++++++++++++++++++++++++++++++-------------- config.json | 3 +- create_libpack.py | 11 ++- 3 files changed, 128 insertions(+), 55 deletions(-) diff --git a/compile_all.py b/compile_all.py index 568fbcb..210b1cc 100644 --- a/compile_all.py +++ b/compile_all.py @@ -80,6 +80,15 @@ def patch_files(patches: list[str]) -> None: apply_patch(patch) +def libpack_dir(config: dict, mode: BuildMode): + dir = "LibPack-{}-v{}-{}".format( + config["FreeCAD-version"], + config["LibPack-version"], + str(mode), + ) + return os.path.join(os.path.dirname(__file__), "working", dir) + + class Compiler: def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.config = config @@ -87,56 +96,53 @@ def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.bison_path = bison_path self.base_dir = os.getcwd() self.skip_existing = skip_existing - libpack_dir = "LibPack-{}-v{}-{}".format( - config["FreeCAD-version"], - config["LibPack-version"], - str(mode), - ) - self.install_dir = os.path.join(os.getcwd(), libpack_dir) + self.install_dir = libpack_dir(config, mode) self.init_script = None def get_cmake_options(self) -> list[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ base = [ - f"-DBISON_EXECUTABLE={self.bison_path}", - f"-DBOOST_ROOT={self.install_dir}", - f"-DBUILD_DOCS=No", - f"-DBUILD_EXAMPLES=No", - f"-DBUILD_SHARED=Yes", - f"-DBUILD_SHARED_LIB=Yes", - f"-DBUILD_SHARED_LIBS=Yes", - f"-DBUILD_TEST=No", - f"-DBUILD_TESTS=No", - f"-DBUILD_TESTING=No", - f"-DBZIP2_DIR={self.install_dir}/lib/cmake/", - f"-DBoost_INCLUDE_DIR={self.install_dir}/include", - f"-DBoost_INCLUDE_DIRS={self.install_dir}/include", - f"-DCMAKE_INSTALL_PATH={self.install_dir}", - f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", - f"-DCoin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", - f"-DHarfBuzz_DIR={self.install_dir}/lib/cmake/", - f"-DHDF5_DIR={self.install_dir}/share/cmake/", - f"-DHDF5_LIBRARY_DEBUG=LIBPACK/lib/hdf5.lib", - f"-DHDF5_LIBRARY_RELEASE=LIBPACK/lib/hdf5.lib", - f"-DHDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + ".exe" if sys.platform.startswith("win32") else "", - f"-DINSTALL_DIR={self.install_dir}", - f"-DPCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", - f"-DPython_ROOT_DIR={self.install_dir}/bin", - f"-DQt6_DIR={self.install_dir}/lib/cmake/Qt6", - f"-DSWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", - f"-DVTK_MODULE_ENABLE_VTK_IOIOSS=NO", # Workaround for bug in Visual Studio MSVC 143 - f"-DVTK_MODULE_ENABLE_VTK_ioss=NO", # Workaround for bug in Visual Studio MSVC 143 - f"-DZLIB_DIR={self.install_dir}/lib/cmake/", - f"-DZLIB_INCLUDE_DIR={self.install_dir}/include", - f"-DZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith("win32") else "a", + f"-D BISON_EXECUTABLE={self.bison_path}", + f"-D BOOST_ROOT={self.install_dir}", + f"-D BUILD_DOCS=No", + f"-D BUILD_EXAMPLES=No", + f"-D BUILD_SHARED=Yes", + f"-D BUILD_SHARED_LIB=Yes", + f"-D BUILD_SHARED_LIBS=Yes", + f"-D BUILD_TEST=No", + f"-D BUILD_TESTS=No", + f"-D BUILD_TESTING=No", + f"-D BZIP2_DIR={self.install_dir}/lib/cmake/", + f"-D Boost_INCLUDE_DIR={self.install_dir}/include", + f"-D Boost_INCLUDE_DIRS={self.install_dir}/include", + f"-D CMAKE_INSTALL_PATH={self.install_dir}", + f"-D CMAKE_INSTALL_PREFIX={self.install_dir}", + f"-D Coin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", + f"-D HarfBuzz_DIR={self.install_dir}/lib/cmake/", + f"-D HDF5_DIR={self.install_dir}/share/cmake/", + f"-D HDF5_LIBRARY_DEBUG=LIBPACK/lib/hdf5.lib", + f"-D HDF5_LIBRARY_RELEASE=LIBPACK/lib/hdf5.lib", + f"-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + ".exe" if sys.platform.startswith( + "win32") else "", + f"-D INSTALL_DIR={self.install_dir}", + f"-D PCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", + f"-D Python_ROOT_DIR={self.install_dir}/bin", + f"-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6", + f"-D SWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", + f"-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-D VTK_MODULE_ENABLE_VTK_ioss=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-D ZLIB_DIR={self.install_dir}/lib/cmake/", + f"-D ZLIB_INCLUDE_DIR={self.install_dir}/include", + f"-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith( + "win32") else "a", ] if sys.platform.startswith("win32"): inc_path = self.install_dir.replace('\\', '/') CXX_FLAGS = f"/I{inc_path}/include /EHsc /DWIN32" else: - CXX_FLAGS= f"-I{self.install_dir}/include" - base.append(f"-DCMAKE_CXX_FLAGS={CXX_FLAGS}") + CXX_FLAGS = f"-I{self.install_dir}/include" + base.append(f"-D CMAKE_CXX_FLAGS={CXX_FLAGS}") return base def compile_all(self): @@ -248,29 +254,50 @@ def build_boost(self, _=None): print(e.output.decode("utf-8")) exit(e.returncode) - def _build_standard_cmake(self, extra_args: list[str] = None): + def _cmake_create_build_dir(self): build_dir = "build-" + str(self.mode).lower() if not os.path.exists(build_dir): os.mkdir(build_dir) os.chdir(build_dir) + def _cmake_configure(self, extra_args: list[str] = None): cmake_setup_options = ["cmake"] standard_options = self.get_cmake_options() cmake_setup_options.extend(standard_options) if extra_args: cmake_setup_options.extend(extra_args) cmake_setup_options.append("..") - cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] - cmake_install_options = ["cmake", "--install", "."] try: subprocess.run(cmake_setup_options, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: cMake Configure failed!") + print(e.output.decode("utf-8")) + exit(e.returncode) + + def _cmake_build(self): + cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] + try: subprocess.run(cmake_build_options, check=True, capture_output=True) - subprocess.run(cmake_install_options, check=True, capture_output=True) except subprocess.CalledProcessError as e: print("ERROR: Build failed!") print(e.output.decode("utf-8")) exit(e.returncode) + def _cmake_install(self): + cmake_install_options = ["cmake", "--install", "."] + try: + subprocess.run(cmake_install_options, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Install failed!") + print(e.output.decode("utf-8")) + exit(e.returncode) + + def _build_standard_cmake(self, extra_args: list[str] = None): + self._cmake_create_build_dir() + self._cmake_configure(extra_args) + self._cmake_build() + self._cmake_install() + def build_coin(self, _=None): """ Builds and installs Coin using standard CMake settings """ if self.skip_existing: @@ -394,6 +421,20 @@ def build_freetype(self, _=None): return self._build_standard_cmake() + def force_copy(self, src_components: list[str], dst_components: list[str]): + full_src = self.install_dir + for src in src_components: + full_src = os.path.join(full_src, src) + full_dst = self.install_dir + for dst in dst_components: + full_dst = os.path.join(full_dst, dst) + if not os.path.exists(full_src): + print(f" (Can't rename {full_src}, no such file or directory)") + return + if os.path.exists(full_dst): + os.unlink(full_dst) + shutil.copyfile(full_src, full_dst) + def build_tcl(self, _=None): """ tcl does not use cMake """ if self.skip_existing: @@ -413,6 +454,9 @@ def build_tcl(self, _=None): "install", f"INSTALLDIR={self.install_dir}"] subprocess.run(args, check=True, capture_output=True) + self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"]) + self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"]) + self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tcl using nmake") print(e.output.decode("utf-8")) @@ -433,8 +477,12 @@ def build_tk(self, _=None): os.chdir("win") args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", f"INSTALLDIR={self.install_dir}"] + args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", + f"INSTALLDIR={self.install_dir}"] subprocess.run(args, check=True, capture_output=True) + self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"]) + self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"]) + self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tk using nmake") print(e.output.decode("utf-8")) @@ -449,8 +497,10 @@ def build_opencascade(self, _=None): if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): print(" Not rebuilding OpenCASCADE, it is already in the LibPack") return - extra_args = [f"-D3RDPARTY_DIR={self.install_dir}", "-DUSE_VTK=ON", - f"-D3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2"] + extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", + f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 + f"-D USE_VTK=On", + f"-D BUILD_CPP_STANDARD=C++17"] if self.mode == BuildMode.DEBUG: extra_args.append("-DBUILD_SHARED_LIBRARY_NAME_POSTFIX=d") self._build_standard_cmake(extra_args=extra_args) @@ -475,10 +525,12 @@ def build_hdf5(self, _: None): self._build_standard_cmake() def build_medfile(self, _: None): - print(" *** Salome MED File source is not currently available -- skipping in this build ***") - return - #extra_args = ["-DMEDFILE_USE_UNICODE=On"] - #self._build_standard_cmake(extra_args) + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "medfile.h")): + print(" Not rebuilding medfile, it is already in the LibPack") + return + extra_args = ["-DMEDFILE_USE_UNICODE=On"] + self._build_standard_cmake(extra_args) def build_gmsh(self, _: None): if self.skip_existing: @@ -513,12 +565,25 @@ def build_pycxx(self, _: None): def build_icu(self, _: None): """ ICU does not use cMake, but has projects for various OSes """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "unicode")): + print(" Not rebuilding ICU, it is already in the LibPack") + return + os.chdir(os.path.join("icu4c", "source")) if sys.platform.startswith("win32"): - args = ["msbuild", f"/p:Configuration={self.mode}", "/t:Build", "allinone.sln"] + os.chdir("allinone") + args = [self.init_script, "&", "msbuild", f"/p:Configuration={self.mode}", + "/t:Build", "/p:SkipUWP=true", "allinone.sln"] subprocess.run(args, check=True, capture_output=True) + bin_dir = os.path.join(self.install_dir, "bin") + lib_dir = os.path.join(bin_dir, "lib") + inc_dir = os.path.join(bin_dir, "include") + os.chdir(os.path.join("..","..")) + shutil.copytree(f"bin64", bin_dir, dirs_exist_ok=True) + shutil.copytree(f"lib64", lib_dir, dirs_exist_ok=True) + shutil.copytree(f"include", inc_dir, dirs_exist_ok=True) else: raise NotImplemented( "Non-Windows compilation of ICU is not implemented yet" ) - diff --git a/config.json b/config.json index ca2570a..7790bb9 100644 --- a/config.json +++ b/config.json @@ -102,7 +102,8 @@ }, { "name":"medfile", - "git-repo":"https://github.com/FedoraScientific/salome-med" + "git-repo":"https://github.com/chennes/med", + "git-ref":"v4.1.1" }, { "name":"gmsh", diff --git a/create_libpack.py b/create_libpack.py index 92b1f7e..56dcfd1 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -1,4 +1,5 @@ #!/bin/python3 +import sys # SPDX-License-Identifier: LGPL-2.1-or-later @@ -170,8 +171,14 @@ def decompress(name: str, filename: str): path_to_7zip = args["7zip"] path_to_bison = args["bison"] - if not os.path.exists(os.path.join("working","bin")): - print("ERROR: ") + base = compile_all.libpack_dir(config, compile_all.BuildMode.RELEASE) + expected_py = os.path.join(base, "bin", "python") + if sys.platform.startswith("win32"): + expected_py += ".exe" + if not os.path.exists(expected_py): + print(f"ERROR: Could not find Python at {expected_py}") + print("Run the bootstrap.py script and then install Python into the created 'bin' directory") + exit(1) os.chdir(args["working"]) fetch_remote_data(config, args["skip_existing_clone"]) From 3c2fe28bb3035b4258e341cd3fcfba0273b9acc7 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Mon, 21 Aug 2023 22:31:04 -0500 Subject: [PATCH 12/27] Add xerces-c, libfmt, and eigen3 --- compile_all.py | 29 ++++++++++++++++++++++++++--- config.json | 2 +- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/compile_all.py b/compile_all.py index 210b1cc..d02ec3b 100644 --- a/compile_all.py +++ b/compile_all.py @@ -105,6 +105,7 @@ def get_cmake_options(self) -> list[str]: base = [ f"-D BISON_EXECUTABLE={self.bison_path}", f"-D BOOST_ROOT={self.install_dir}", + f"-D BUILD_DOC=No", f"-D BUILD_DOCS=No", f"-D BUILD_EXAMPLES=No", f"-D BUILD_SHARED=Yes", @@ -577,9 +578,9 @@ def build_icu(self, _: None): "/t:Build", "/p:SkipUWP=true", "allinone.sln"] subprocess.run(args, check=True, capture_output=True) bin_dir = os.path.join(self.install_dir, "bin") - lib_dir = os.path.join(bin_dir, "lib") - inc_dir = os.path.join(bin_dir, "include") - os.chdir(os.path.join("..","..")) + lib_dir = os.path.join(self.install_dir, "lib") + inc_dir = os.path.join(self.install_dir, "include") + os.chdir(os.path.join("..", "..")) shutil.copytree(f"bin64", bin_dir, dirs_exist_ok=True) shutil.copytree(f"lib64", lib_dir, dirs_exist_ok=True) shutil.copytree(f"include", inc_dir, dirs_exist_ok=True) @@ -587,3 +588,25 @@ def build_icu(self, _: None): raise NotImplemented( "Non-Windows compilation of ICU is not implemented yet" ) + + def build_xercesc(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "xercesc")): + print(" Not rebuilding xerces-c, it is already in the LibPack") + return + extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include/unicode"] + self._build_standard_cmake(extra_args) + + def build_libfmt(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "fmt")): + print(" Not rebuilding libfmt, it is already in the LibPack") + return + self._build_standard_cmake() + + def build_eigen3(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "eigen3")): + print(" Not rebuilding Eigen3, it is already in the LibPack") + return + self._build_standard_cmake() diff --git a/config.json b/config.json index 7790bb9..3dc7729 100644 --- a/config.json +++ b/config.json @@ -121,7 +121,7 @@ "git-ref":"release-73-2" }, { - "name":"xerces-c", + "name":"xercesc", "git-repo":"https://github.com/apache/xerces-c", "git-ref":"v3.2.4" }, From 859af366c882114b2c30e258eb1106353518d1fa Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Mon, 21 Aug 2023 23:18:51 -0500 Subject: [PATCH 13/27] Add version file that cmake can read --- create_libpack.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/create_libpack.py b/create_libpack.py index 56dcfd1..438b68a 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -126,6 +126,15 @@ def decompress(name: str, filename: str): os.chdir(original_dir) +def write_manifest(outer_config: dict, mode): + manifest_file = os.path.join(compile_all.libpack_dir(outer_config, mode), "manifest.json") + with open(manifest_file, "w", encoding="utf-8") as f: + f.write(json.dumps(outer_config["content"])) + version_file = os.path.join(compile_all.libpack_dir(outer_config, mode), "FREECAD_LIBPACK_VERSION") + with open(version_file, "w", encoding="utf-8") as f: + f.write(outer_config["LibPack-version"]) + + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Builds a collection of FreeCAD dependencies for the current system" @@ -192,7 +201,5 @@ def decompress(name: str, filename: str): compiler.init_script = devel_init_script compiler.compile_all() + write_manifest(config, compile_all.BuildMode.RELEASE) -# Preliminary setup that will be needed for running CMake -# CMAKE_PREFIX_PATH to libpack dir -# CMAKE_INSTALL_PREFIX to libpack dir From 6c1601ef1e72f126849c082474e402508f7c77ab Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Thu, 24 Aug 2023 17:59:26 -0500 Subject: [PATCH 14/27] Refine Netgen cMake flags --- compile_all.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/compile_all.py b/compile_all.py index d02ec3b..636f8e0 100644 --- a/compile_all.py +++ b/compile_all.py @@ -511,11 +511,16 @@ def build_netgen(self, _: None): if os.path.exists(os.path.join(self.install_dir, "share", "netgen")): print(" Not rebuilding netgen, it is already in the LibPack") return - extra_args = ["-D USE_SUPERBUILD=OFF", + extra_args = [f"-DCMAKE_FIND_ROOT_PATH={self.install_dir}", + "-D USE_SUPERBUILD=OFF", "-D USE_GUI=OFF", - f"-D USE_INTERNAL_TCL=OFF", + "-D USE_INTERNAL_TCL=OFF", f"-D TCL_DIR={self.install_dir}", - f"-D TK_DIR={self.install_dir}"] + f"-D TK_DIR={self.install_dir}", + "-D USE_OCC=On", + f"-D OpenCASCADE_ROOT={self.install_dir}", + f"-D USE_PYTHON=On", + f"-D PYTHON_EXECUTABLE={self.install_dir}/bin/python{'.exe' if sys.platform.startswith('win32') else ''}"] self._build_standard_cmake(extra_args=extra_args) def build_hdf5(self, _: None): From c6c489da3bc4f147a405f2cdc05875e5da0a8a4f Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 26 Aug 2023 00:07:45 -0500 Subject: [PATCH 15/27] Bugfixes and more Python deps --- bootstrap.py | 2 +- compile_all.py | 108 +++++++++++++++++++++++++++++++++------------- config.json | 33 ++++++++++++++ create_libpack.py | 13 +++++- 4 files changed, 123 insertions(+), 33 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index cfe21d5..f061a11 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -62,7 +62,7 @@ def create_libpack_dir(config: dict, mode: BuildMode) -> str: os.rename(dirname, backup_name) if not os.path.exists(dirname): os.mkdir(dirname) - dirname = os.pawth.join(dirname, "bin") + dirname = os.path.join(dirname, "bin") if not os.path.exists(dirname): os.mkdir(dirname) return dirname diff --git a/compile_all.py b/compile_all.py index 636f8e0..25b8873 100644 --- a/compile_all.py +++ b/compile_all.py @@ -76,17 +76,27 @@ def patch_files(patches: list[str]) -> None: expected to be given as paths relative to **this** Python script file""" for patch in patches: start = len("patches/") - print(f"Applying patch {patch[start:]}") + print(f" Applying patch {patch[start:]}") apply_patch(patch) def libpack_dir(config: dict, mode: BuildMode): - dir = "LibPack-{}-v{}-{}".format( + lp_dir = "LibPack-{}-v{}-{}".format( config["FreeCAD-version"], config["LibPack-version"], str(mode), ) - return os.path.join(os.path.dirname(__file__), "working", dir) + return os.path.join(os.path.dirname(__file__), "working", lp_dir) + + +def cmake_install(): + cmake_install_options = ["cmake", "--install", "."] + try: + subprocess.run(cmake_install_options, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print("ERROR: Install failed!") + print(e.output.decode("utf-8")) + exit(e.returncode) class Compiler: @@ -140,10 +150,10 @@ def get_cmake_options(self) -> list[str]: ] if sys.platform.startswith("win32"): inc_path = self.install_dir.replace('\\', '/') - CXX_FLAGS = f"/I{inc_path}/include /EHsc /DWIN32" + cxx_flags = f"/I{inc_path}/include /EHsc /DWIN32" else: - CXX_FLAGS = f"-I{self.install_dir}/include" - base.append(f"-D CMAKE_CXX_FLAGS={CXX_FLAGS}") + cxx_flags = f"-I{self.install_dir}/include" + base.append(f"-D CMAKE_CXX_FLAGS={cxx_flags}") return base def compile_all(self): @@ -239,14 +249,19 @@ def build_boost(self, _=None): exe = os.path.join(self.install_dir, "bin", "python") if sys.platform.startswith("win32"): exe += ".exe" - inc_dir = os.path.join(self.install_dir, "bin", "Include") - lib_dir = os.path.join(self.install_dir, "bin", "Lib") + exe = exe.replace("\\", "\\\\") + inc_dir = os.path.join(self.install_dir, "bin", "include").replace("\\", "\\\\") + lib_dir = os.path.join(self.install_dir, "bin", "libs").replace("\\", "\\\\") python_version = self.get_python_version() print(f"Building boost-python with Python {python_version}") user_config.write(f'using python : {python_version} : "{exe}" : "{inc_dir}" : "{lib_dir}" ;\n') try: + # When debugging on the command line, add --debug-configuration to get more verbose output subprocess.run(["bootstrap.bat"], capture_output=True, check=True) - subprocess.run(["b2", f"variant={str(self.mode).lower()}"], check=True, capture_output=True) + subprocess.run(["b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=static"], check=True, + capture_output=True) + subprocess.run(["b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=shared"], check=True, + capture_output=True) shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), dirs_exist_ok=True) @@ -284,20 +299,50 @@ def _cmake_build(self): print(e.output.decode("utf-8")) exit(e.returncode) - def _cmake_install(self): - cmake_install_options = ["cmake", "--install", "."] - try: - subprocess.run(cmake_install_options, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print("ERROR: Install failed!") - print(e.output.decode("utf-8")) - exit(e.returncode) - def _build_standard_cmake(self, extra_args: list[str] = None): self._cmake_create_build_dir() self._cmake_configure(extra_args) self._cmake_build() - self._cmake_install() + cmake_install() + + def _pip_install(self, requirement: str) -> None: + path_to_python = os.path.join(self.install_dir, "bin", "python") + if sys.platform.startswith("win32"): + path_to_python += ".exe" + try: + subprocess.run([path_to_python, "-m", "pip", "install", requirement], check=True, + capture_output=True) + except subprocess.CalledProcessError as e: + print(f"ERROR: Failed to pip install {requirement}") + print(e.output.decode("utf-8")) + exit(1) + + def _build_with_pip(self, options: dict): + if "pip-install" not in options: + print(f"ERROR: No pip-install provided in configuration of {options['name']}, so version cannot be determined") + exit(1) + self._pip_install(options["pip-install"]) + + def build_numpy(self, options=None): + self._build_with_pip(options) + + def build_scipy(self, options=None): + self._build_with_pip(options) + + def build_pillow(self, options=None): + self._build_with_pip(options) + + def build_pyyaml(self, options=None): + self._build_with_pip(options) + + def build_pycollada(self, options=None): + self._build_with_pip(options) + + def build_matplotlib(self, options=None): + self._build_with_pip(options) + + def build_opencv(self, options=None): + self._build_with_pip(options) def build_coin(self, _=None): """ Builds and installs Coin using standard CMake settings """ @@ -305,7 +350,8 @@ def build_coin(self, _=None): if os.path.exists(os.path.join(self.install_dir, "share", "Coin")): print(" Not rebuilding Coin, it is already in the LibPack") return - self._build_standard_cmake() + extra_args = ["-DCOIN_BUILD_TESTS=Off"] + self._build_standard_cmake(extra_args) def build_quarter(self, _=None): """ Builds and installs Quarter using standard CMake settings """ @@ -383,16 +429,7 @@ def build_pyside(self, options=None): if "pip-install" not in options: print("ERROR: No pip-install provided in configuration of pyside, so version cannot be determined") exit(1) - path_to_python = os.path.join(self.install_dir, "bin", "python") - if sys.platform.startswith("win32"): - path_to_python += ".exe" - try: - subprocess.run([path_to_python, "-m", "pip", "install", options['pip-install']], check=True, - capture_output=True) - except subprocess.CalledProcessError as e: - print(f"ERROR: Failed to pip install {options['pip-install']}") - print(e.output.decode("utf-8")) - exit(1) + self._pip_install(options["pip-install"]) def build_vtk(self, _=None): if self.skip_existing: @@ -520,7 +557,8 @@ def build_netgen(self, _: None): "-D USE_OCC=On", f"-D OpenCASCADE_ROOT={self.install_dir}", f"-D USE_PYTHON=On", - f"-D PYTHON_EXECUTABLE={self.install_dir}/bin/python{'.exe' if sys.platform.startswith('win32') else ''}"] + "-D PYTHON_EXECUTABLE=" + f"{self.install_dir}/bin/python{'.exe' if sys.platform.startswith('win32') else ''}"] self._build_standard_cmake(extra_args=extra_args) def build_hdf5(self, _: None): @@ -615,3 +653,11 @@ def build_eigen3(self, _: None): print(" Not rebuilding Eigen3, it is already in the LibPack") return self._build_standard_cmake() + + def build_yamlcpp(self, _: None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "yaml-cpp")): + print(" Not rebuilding yaml-cpp, it is already in the LibPack") + return + extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"] + self._build_standard_cmake((extra_args)) \ No newline at end of file diff --git a/config.json b/config.json index 3dc7729..92f6c20 100644 --- a/config.json +++ b/config.json @@ -6,6 +6,34 @@ "name":"qt", "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" }, + { + "name":"numpy", + "pip-install":"numpy==1.25.2" + }, + { + "name":"scipy", + "pip-install":"scipy==1.11.2" + }, + { + "name":"pillow", + "pip-install":"Pillow==10.0.0" + }, + { + "name":"pyyaml", + "pip-install":"PyYAML==6.0.1" + }, + { + "name":"pycollada", + "pip-install":"pycollada==0.7.2" + }, + { + "name":"matplotlib", + "pip-install":"matplotlib==3.7.2" + }, + { + "name":"opencv", + "pip-install":"opencv-python==4.8.0.76" + }, { "name":"boost", "git-repo":"https://github.com/boostorg/boost", @@ -134,6 +162,11 @@ "name":"eigen3", "git-repo":"https://gitlab.com/libeigen/eigen", "git-ref":"3.4.0" + }, + { + "name": "yamlcpp", + "git-repo": "https://github.com/jbeder/yaml-cpp", + "git-ref":"0.8.0" } ] } diff --git a/create_libpack.py b/create_libpack.py index 438b68a..5e7a86f 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -21,9 +21,20 @@ import shutil import stat import subprocess -import requests from urllib.parse import urlparse +try: + import requests +except ImportError: + print("Please pip --install requests") + exit(1) + +try: + import diff_match_patch +except ImportError: + print("Please pip --install diff_match_patch") + exit(1) + import compile_all path_to_7zip = "C:\\Program Files\\7-Zip\\7z.exe" From cb9a858817482a935d49349349373162ae73623e Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 26 Aug 2023 13:54:03 -0500 Subject: [PATCH 16/27] Flags refinement, general refactoring, add yaml-cpp --- compile_all.py | 57 ++++++++----------- config.json | 3 +- create_libpack.py | 2 + ...ler_bug_workaround_msvc14_std_atomic.patch | 22 +++++++ 4 files changed, 51 insertions(+), 33 deletions(-) create mode 100644 patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch diff --git a/compile_all.py b/compile_all.py index 25b8873..b1f089d 100644 --- a/compile_all.py +++ b/compile_all.py @@ -89,16 +89,6 @@ def libpack_dir(config: dict, mode: BuildMode): return os.path.join(os.path.dirname(__file__), "working", lp_dir) -def cmake_install(): - cmake_install_options = ["cmake", "--install", "."] - try: - subprocess.run(cmake_install_options, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print("ERROR: Install failed!") - print(e.output.decode("utf-8")) - exit(e.returncode) - - class Compiler: def __init__(self, config, mode, bison_path, skip_existing: bool = False): self.config = config @@ -161,8 +151,6 @@ def compile_all(self): # All build methods are named using "build_XXX" where XXX is the name of the package in the config file print(f"Building {item['name']} in {self.mode} mode") os.chdir(item["name"]) - if "patches" in item: - patch_files(item["patches"]) build_function_name = "build_" + item["name"] build_function = getattr(self, build_function_name) build_function(item) @@ -183,6 +171,7 @@ def build_python(self, _=None): path = "amd64" if platform.machine() == "AMD64" else "arm64" subprocess.run( [ + self.init_script, "&", "PCbuild\\build.bat", "-p", arch, @@ -257,10 +246,10 @@ def build_boost(self, _=None): user_config.write(f'using python : {python_version} : "{exe}" : "{inc_dir}" : "{lib_dir}" ;\n') try: # When debugging on the command line, add --debug-configuration to get more verbose output - subprocess.run(["bootstrap.bat"], capture_output=True, check=True) - subprocess.run(["b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=static"], check=True, + subprocess.run([self.init_script, "&", "bootstrap.bat"], capture_output=True, check=True) + subprocess.run([self.init_script, "&", "b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=static"], check=True, capture_output=True) - subprocess.run(["b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=shared"], check=True, + subprocess.run([self.init_script, "&", "b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=shared"], check=True, capture_output=True) shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), @@ -276,34 +265,36 @@ def _cmake_create_build_dir(self): os.mkdir(build_dir) os.chdir(build_dir) - def _cmake_configure(self, extra_args: list[str] = None): - cmake_setup_options = ["cmake"] - standard_options = self.get_cmake_options() - cmake_setup_options.extend(standard_options) - if extra_args: - cmake_setup_options.extend(extra_args) - cmake_setup_options.append("..") + def _run_cmake(self, args): + cmake_setup_options = [self.init_script, "&", "cmake"] + cmake_setup_options.extend(args) try: subprocess.run(cmake_setup_options, check=True, capture_output=True) except subprocess.CalledProcessError as e: - print("ERROR: cMake Configure failed!") + print("ERROR: cMake failed!") print(e.output.decode("utf-8")) exit(e.returncode) + def _cmake_configure(self, extra_args: list[str] = None): + options = self.get_cmake_options() + if extra_args: + options.extend(extra_args) + options.append("..") + self._run_cmake(options) + def _cmake_build(self): - cmake_build_options = ["cmake", "--build", ".", "--config", str(self.mode), "--parallel"] - try: - subprocess.run(cmake_build_options, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print("ERROR: Build failed!") - print(e.output.decode("utf-8")) - exit(e.returncode) + cmake_build_options = ["--build", ".", "--config", str(self.mode), "--parallel"] + self._run_cmake(cmake_build_options) + + def _cmake_install(self): + cmake_install_options = ["--install", "."] + self._run_cmake(cmake_install_options) def _build_standard_cmake(self, extra_args: list[str] = None): self._cmake_create_build_dir() self._cmake_configure(extra_args) self._cmake_build() - cmake_install() + self._cmake_install() def _pip_install(self, requirement: str) -> None: path_to_python = os.path.join(self.install_dir, "bin", "python") @@ -637,7 +628,9 @@ def build_xercesc(self, _: None): if os.path.exists(os.path.join(self.install_dir, "include", "xercesc")): print(" Not rebuilding xerces-c, it is already in the LibPack") return - extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include/unicode"] + extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include/unicode", + f"-D ICU_ROOT={self.install_dir}", + f"-D ICU_UC_DIR={self.install_dir}"] self._build_standard_cmake(extra_args) def build_libfmt(self, _: None): diff --git a/config.json b/config.json index 92f6c20..642778f 100644 --- a/config.json +++ b/config.json @@ -121,7 +121,8 @@ { "name":"netgen", "git-repo":"https://github.com/NGSolve/netgen", - "git-ref":"v6.2.2304" + "git-ref":"v6.2.2304", + "patches":["patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch"] }, { "name":"hdf5", diff --git a/create_libpack.py b/create_libpack.py index 5e7a86f..9473dd8 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -93,6 +93,8 @@ def fetch_remote_data(config: dict, skip_existing: bool = False): else: # Just make the directory, presumably later code will know what to do os.makedirs(item["name"], exist_ok=True) + if "patches" in item: + compile_all.patch_files(item["patches"]) def clone(name: str, url: str, ref: str = None): diff --git a/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch b/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch new file mode 100644 index 0000000..38814e9 --- /dev/null +++ b/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch @@ -0,0 +1,22 @@ +@@@ libsrc/meshing/meshclass.cpp @@@ +@@ -127560,24 +127560,139 @@ + ocal_sum);%0A%0A ++ // NOTE: patched with a simple refactor to work around a compiler bug in MSVC 14.3%0A std::mutex m;%0A + for +@@ -127715,24 +127715,26 @@ + (n_classes)) ++ %7B + %0A +@@ -127739,17 +127739,44 @@ + +-AsAtomic( ++std::scoped_lock%7B m %7D;%0A + tets +@@ -127791,17 +127791,16 @@ + class%5Bi%5D +-) + += clas +@@ -127809,24 +127809,35 @@ + s_local%5Bi%5D;%0A ++ %7D%0A + %7D);%0A%0A From bfc7adb50a1e6dfdf4a0200c265c2348606a2403 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 27 Aug 2023 10:15:15 -0500 Subject: [PATCH 17/27] Further refinement to options --- compile_all.py | 29 ++++++--- config.json | 30 +++++----- create_libpack.py | 3 + ...ler_bug_workaround_msvc14_std_atomic.patch | 60 ++++++++++++------- 4 files changed, 78 insertions(+), 44 deletions(-) diff --git a/compile_all.py b/compile_all.py index b1f089d..324973b 100644 --- a/compile_all.py +++ b/compile_all.py @@ -115,7 +115,7 @@ def get_cmake_options(self) -> list[str]: f"-D BUILD_TESTS=No", f"-D BUILD_TESTING=No", f"-D BZIP2_DIR={self.install_dir}/lib/cmake/", - f"-D Boost_INCLUDE_DIR={self.install_dir}/include", + f"-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/", f"-D Boost_INCLUDE_DIRS={self.install_dir}/include", f"-D CMAKE_INSTALL_PATH={self.install_dir}", f"-D CMAKE_INSTALL_PREFIX={self.install_dir}", @@ -242,18 +242,24 @@ def build_boost(self, _=None): inc_dir = os.path.join(self.install_dir, "bin", "include").replace("\\", "\\\\") lib_dir = os.path.join(self.install_dir, "bin", "libs").replace("\\", "\\\\") python_version = self.get_python_version() - print(f"Building boost-python with Python {python_version}") + print(f" (boost-python is being built against Python {python_version})") user_config.write(f'using python : {python_version} : "{exe}" : "{inc_dir}" : "{lib_dir}" ;\n') try: # When debugging on the command line, add --debug-configuration to get more verbose output subprocess.run([self.init_script, "&", "bootstrap.bat"], capture_output=True, check=True) - subprocess.run([self.init_script, "&", "b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=static"], check=True, + subprocess.run([self.init_script, "&", "b2", + f"install", + f"variant={str(self.mode).lower()}", + "address-model=64", + "link=shared", + f"--prefix=${self.install_dir}", + "--layout=versioned", # or system + "--build-type=complete"], # or minimal + check=True, capture_output=True) - subprocess.run([self.init_script, "&", "b2", f"variant={str(self.mode).lower()}", "address-model=64", "link=shared"], check=True, - capture_output=True) - shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) - shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), - dirs_exist_ok=True) + #shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) + #shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), + # dirs_exist_ok=True) except subprocess.CalledProcessError as e: print("Error: failed to build boost") print(e.output.decode("utf-8")) @@ -283,7 +289,8 @@ def _cmake_configure(self, extra_args: list[str] = None): self._run_cmake(options) def _cmake_build(self): - cmake_build_options = ["--build", ".", "--config", str(self.mode), "--parallel"] + # Not building with --parallel because even with 32gb of RAM, OpenCASCADE runs out of memory on my system + cmake_build_options = ["--build", ".", "--config", str(self.mode)] self._run_cmake(cmake_build_options) def _cmake_install(self): @@ -409,6 +416,7 @@ def build_libclang(self, _=None): if os.path.exists(os.path.join(self.install_dir, "include", "clang")): print(" Not copying libclang, it is already in the LibPack") return + print(" (not really building libclang, just copying from a build provided by Qt)") shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True) def build_pyside(self, options=None): @@ -427,6 +435,7 @@ def build_vtk(self, _=None): if os.path.exists(os.path.join(self.install_dir, "share", "licenses", "VTK")): print(" Not rebuilding VTK, it is already in the LibPack") return + print(" (VTK is big, this will take some time)") self._build_standard_cmake() def build_harfbuzz(self, _=None): @@ -557,6 +566,8 @@ def build_hdf5(self, _: None): if os.path.exists(os.path.join(self.install_dir, "include", "hdf5.h")): print(" Not rebuilding hdf5, it is already in the LibPack") return + # Something goes wrong with the install during this script, but when run from the command line it succeeds + # without a problem. self._build_standard_cmake() def build_medfile(self, _: None): diff --git a/config.json b/config.json index 642778f..4fb9494 100644 --- a/config.json +++ b/config.json @@ -34,6 +34,21 @@ "name":"opencv", "pip-install":"opencv-python==4.8.0.76" }, + { + "name":"zlib", + "git-repo":"https://github.com/madler/zlib", + "git-ref":"v1.3" + }, + { + "name":"bzip2", + "git-repo":"https://gitlab.com/bzip2/bzip2.git", + "git-ref":"bzip2-1.0.8" + }, + { + "name":"libpng", + "git-repo":"https://github.com/glennrp/libpng", + "git-ref":"v1.6.40" + }, { "name":"boost", "git-repo":"https://github.com/boostorg/boost", @@ -50,16 +65,6 @@ "git-ref":"master", "patches":["patches/quarter-01-add-QOpenGLContext-to-QuarterWidgetP.patch"] }, - { - "name":"zlib", - "git-repo":"https://github.com/madler/zlib", - "git-ref":"v1.3" - }, - { - "name":"bzip2", - "git-repo":"https://gitlab.com/bzip2/bzip2.git", - "git-ref":"bzip2-1.0.8" - }, { "name":"pcre2", "git-repo":"https://github.com/PCRE2Project/pcre2", @@ -93,11 +98,6 @@ "git-repo":"https://github.com/harfbuzz/harfbuzz", "git-ref":"8.1.1" }, - { - "name":"libpng", - "git-repo":"https://github.com/glennrp/libpng", - "git-ref":"v1.6.40" - }, { "name":"freetype", "git-repo":"https://gitlab.freedesktop.org/freetype/freetype/", diff --git a/create_libpack.py b/create_libpack.py index 9473dd8..8bb1921 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -94,7 +94,10 @@ def fetch_remote_data(config: dict, skip_existing: bool = False): # Just make the directory, presumably later code will know what to do os.makedirs(item["name"], exist_ok=True) if "patches" in item: + cwd = os.getcwd() + os.chdir(item["name"]) compile_all.patch_files(item["patches"]) + os.chdir(cwd) def clone(name: str, url: str, ref: str = None): diff --git a/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch b/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch index 38814e9..3842e2b 100644 --- a/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch +++ b/patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch @@ -1,22 +1,42 @@ @@@ libsrc/meshing/meshclass.cpp @@@ -@@ -127560,24 +127560,139 @@ - ocal_sum);%0A%0A -+ // NOTE: patched with a simple refactor to work around a compiler bug in MSVC 14.3%0A std::mutex m;%0A - for -@@ -127715,24 +127715,26 @@ - (n_classes)) -+ %7B - %0A -@@ -127739,17 +127739,44 @@ +@@ -126921,265 +126921,182 @@ --AsAtomic( -+std::scoped_lock%7B m %7D;%0A - tets -@@ -127791,17 +127791,16 @@ - class%5Bi%5D --) - += clas -@@ -127809,24 +127809,35 @@ - s_local%5Bi%5D;%0A -+ %7D%0A - %7D);%0A%0A +-ParallelForRange( IntRange(volelements.Size()), %5B&%5D (auto myrange)%0A %7B%0A double local_sum = 0.0;%0A double teterrpow = mp.opterrpow;%0A%0A std::array%3Cint,n_classes%3E classes_local%7B%7D;%0A%0A for (auto i : myrange)%0A %7B%0A ++// NOTE: Patched to eliminate the ParallelForRange, which gives MSVC 14.3 an internal compiler error%0A for (auto &e : volelements) %7B%0A double teterrpow = mp.opterrpow;%0A + +@@ -127139,30 +127139,17 @@ + points, +-volelements%5Bi%5D ++e + , 0, mp) +@@ -127174,31 +127174,24 @@ + ow);%0A%0A +- +- + int qualclas +@@ -127219,31 +127219,24 @@ + elbad + 1);%0A +- + if (qu +@@ -127259,31 +127259,24 @@ + lclass = 1;%0A +- + if (qu +@@ -127329,28 +127329,25 @@ + +- classes_local ++tets_in_qualclass + %5Bqua +@@ -127356,35 +127356,19 @@ + lass +--1 + %5D++;%0A +-%0A + +- local_ + sum +@@ -127385,168 +127385,9 @@ + +- %7D%0A%0A AtomicAdd(sum, local_sum);%0A%0A for (auto i : Range(n_classes))%0A AsAtomic(tets_in_qualclass%5Bi%5D) += classes_local%5Bi%5D;%0A %7D); ++%7D + %0A%0A From dc7d30e2f83e53c8f314c0f6555a6807b8609dd2 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 1 Sep 2023 16:47:41 -0500 Subject: [PATCH 18/27] Add builds of release and debug simultaneously --- bootstrap.py | 27 ++-- compile_all.py | 322 +++++++++++++++++++++++++++------------------- config.json | 20 ++- create_libpack.py | 29 +++-- 4 files changed, 235 insertions(+), 163 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index f061a11..3510777 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -6,15 +6,13 @@ import os import sys -from compile_all import BuildMode - """ """ def parse_args() -> dict: if len(sys.argv) > 3: usage() exit(1) - new_config_dict = {"mode": BuildMode.RELEASE, "config_file": "config.json"} + new_config_dict = {"config_file": "config.json"} for arg in sys.argv[1:]: key, value = extract_arg(arg) new_config_dict[key] = value @@ -22,32 +20,29 @@ def parse_args() -> dict: def extract_arg(arg) -> tuple[str, object]: - if arg.lower() in ["release", "debug"]: - return "mode", BuildMode.RELEASE if arg.lower() == "release" else BuildMode.DEBUG return "config_file", arg def usage(): print("Used to create the base LibPack directory that you will then manually install Python into") - print("Usage: python bootstrap.py [config_file] [release|debug]") + print("Usage: python bootstrap.py [config_file]") print() - print('Result: A new working/LibPack-XX-YY-MM directory has been created') + print('Result: A new working/LibPack-XX-YY directory has been created') def print_next_step(): - print('Next step: install Python into working/LibPack-XX-YY-MM/bin, then run:') - print(' .\\working\\LibPack-XX-YY-MM\\bin\\python create_libpack.py') - print('(where XX, YY, and MM change according to the config and inputs)') + print('Next step: install Python into working/LibPack-XX-YY/bin, then run:') + print(' .\\working\\LibPack-XX-YY\\bin\\python create_libpack.py') + print('(where XX, YY change according to the config)') -def create_libpack_dir(config: dict, mode: BuildMode) -> str: +def create_libpack_dir(config: dict) -> str: """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of - the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. + the LibPack. Returns the name of the created directory. """ - dirname = "LibPack-{}-v{}-{}".format( + dirname = "LibPack-{}-v{}".format( config["FreeCAD-version"], - config["LibPack-version"], - str(mode), + config["LibPack-version"] ) if os.path.exists(dirname): backup_name = dirname + "-backup-" + "a" @@ -75,5 +70,5 @@ def create_libpack_dir(config: dict, mode: BuildMode) -> str: config_dict = json.loads(config_data) os.makedirs("working", exist_ok=True) os.chdir("working") - create_libpack_dir(config_dict, args["mode"]) + create_libpack_dir(config_dict) print_next_step() \ No newline at end of file diff --git a/compile_all.py b/compile_all.py index 324973b..902433c 100644 --- a/compile_all.py +++ b/compile_all.py @@ -13,6 +13,7 @@ import re import shutil import subprocess +import stat import sys @@ -29,6 +30,13 @@ def __str__(self) -> str: return "Unknown" +def remove_readonly(func, path, _) -> None: + """Remove a read-only file.""" + + os.chmod(path, stat.S_IWRITE) + func(path) + + def patch_single_file(filename, patch_data) -> None: with open(filename, "r", encoding="utf-8") as f: original_data = f.read() @@ -80,80 +88,88 @@ def patch_files(patches: list[str]) -> None: apply_patch(patch) -def libpack_dir(config: dict, mode: BuildMode): - lp_dir = "LibPack-{}-v{}-{}".format( +def libpack_dir(config: dict): + lp_dir = "LibPack-{}-v{}".format( config["FreeCAD-version"], config["LibPack-version"], - str(mode), ) return os.path.join(os.path.dirname(__file__), "working", lp_dir) class Compiler: - def __init__(self, config, mode, bison_path, skip_existing: bool = False): + def __init__(self, config, bison_path, skip_existing: bool = False): self.config = config - self.mode = mode self.bison_path = bison_path self.base_dir = os.getcwd() self.skip_existing = skip_existing - self.install_dir = libpack_dir(config, mode) + self.install_dir = libpack_dir(config) self.init_script = None def get_cmake_options(self) -> list[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ base = [ - f"-D BISON_EXECUTABLE={self.bison_path}", - f"-D BOOST_ROOT={self.install_dir}", - f"-D BUILD_DOC=No", - f"-D BUILD_DOCS=No", - f"-D BUILD_EXAMPLES=No", - f"-D BUILD_SHARED=Yes", - f"-D BUILD_SHARED_LIB=Yes", - f"-D BUILD_SHARED_LIBS=Yes", - f"-D BUILD_TEST=No", - f"-D BUILD_TESTS=No", - f"-D BUILD_TESTING=No", - f"-D BZIP2_DIR={self.install_dir}/lib/cmake/", - f"-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/", - f"-D Boost_INCLUDE_DIRS={self.install_dir}/include", - f"-D CMAKE_INSTALL_PATH={self.install_dir}", - f"-D CMAKE_INSTALL_PREFIX={self.install_dir}", - f"-D Coin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1", - f"-D HarfBuzz_DIR={self.install_dir}/lib/cmake/", - f"-D HDF5_DIR={self.install_dir}/share/cmake/", - f"-D HDF5_LIBRARY_DEBUG=LIBPACK/lib/hdf5.lib", - f"-D HDF5_LIBRARY_RELEASE=LIBPACK/lib/hdf5.lib", - f"-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + ".exe" if sys.platform.startswith( - "win32") else "", - f"-D INSTALL_DIR={self.install_dir}", - f"-D PCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib", - f"-D Python_ROOT_DIR={self.install_dir}/bin", - f"-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6", - f"-D SWIG_EXECUTABLE={self.install_dir}/bin/swig" + ".exe" if sys.platform.startswith("win32") else "", - f"-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO", # Workaround for bug in Visual Studio MSVC 143 - f"-D VTK_MODULE_ENABLE_VTK_ioss=NO", # Workaround for bug in Visual Studio MSVC 143 - f"-D ZLIB_DIR={self.install_dir}/lib/cmake/", - f"-D ZLIB_INCLUDE_DIR={self.install_dir}/include", - f"-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib." + "lib" if sys.platform.startswith( - "win32") else "a", + f'-D BISON_EXECUTABLE={self.bison_path}', + f'-D BOOST_ROOT={self.install_dir}', + f'-D BUILD_DOC=No', + f'-D BUILD_DOCS=No', + f'-D BUILD_EXAMPLES=No', + f'-D BUILD_SHARED=Yes', + f'-D BUILD_SHARED_LIB=Yes', + f'-D BUILD_SHARED_LIBS=Yes', + f'-D BUILD_TEST=No', + f'-D BUILD_TESTS=No', + f'-D BUILD_TESTING=No', + f'-D BZIP2_DIR={self.install_dir}/lib/cmake/', + f'-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/', + f'-D Boost_INCLUDE_DIRS={self.install_dir}/include', + f'-D CMAKE_INSTALL_PATH={self.install_dir}', + f'-D CMAKE_INSTALL_PREFIX={self.install_dir}', + f'-D Coin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1', + f'-D HarfBuzz_DIR={self.install_dir}/lib/cmake/', + f'-D HDF5_DIR={self.install_dir}/share/cmake/', + f'-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib', + f'-D HDF5_LIBRARY_RELEASE={self.install_dir}/lib/hdf5.lib', + f'-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff' + '.exe' if sys.platform.startswith( + 'win32') else '', + f'-D INSTALL_DIR={self.install_dir}', + f'-D PCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib', + f'-D Python_ROOT_DIR={self.install_dir}/bin', + f'-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6', + f'-D SWIG_EXECUTABLE={self.install_dir}/bin/swig' + '.exe' if sys.platform.startswith('win32') else '', + f'-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO', # Workaround for bug in Visual Studio MSVC 143 + f'-D VTK_MODULE_ENABLE_VTK_ioss=NO', # Workaround for bug in Visual Studio MSVC 143 + f'-D ZLIB_DIR={self.install_dir}/lib/cmake/', + f'-D ZLIB_INCLUDE_DIR={self.install_dir}/include', + f'-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib.' + 'lib' if sys.platform.startswith( + 'win32') else 'a', + f'-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd.' + 'lib' if sys.platform.startswith( + 'win32') else 'a', ] - if sys.platform.startswith("win32"): + if sys.platform.startswith('win32'): inc_path = self.install_dir.replace('\\', '/') - cxx_flags = f"/I{inc_path}/include /EHsc /DWIN32" + cxx_flags = f'/I{inc_path}/include /EHsc /DWIN32' else: - cxx_flags = f"-I{self.install_dir}/include" - base.append(f"-D CMAKE_CXX_FLAGS={cxx_flags}") + cxx_flags = f'-I{self.install_dir}/include' + base.append(f'-D CMAKE_CXX_FLAGS={cxx_flags}') return base def compile_all(self): for item in self.config["content"]: # All build methods are named using "build_XXX" where XXX is the name of the package in the config file - print(f"Building {item['name']} in {self.mode} mode") os.chdir(item["name"]) build_function_name = "build_" + item["name"] - build_function = getattr(self, build_function_name) - build_function(item) + if hasattr(self, build_function_name): + print(f"Building {item['name']}") + build_function = getattr(self, build_function_name) + build_function(item) + elif "pip-install" in item: + print(f"Installing {item['name']} with pip") + self._build_with_pip(item) + else: + print(f"No '{build_function_name}' found in compile_all.py -- " + "did you forget to add one when adding a dependency?") + exit(2) os.chdir(self.base_dir) def build_nonexistent(self, _=None): @@ -176,7 +192,7 @@ def build_python(self, _=None): "-p", arch, "-c", - str(self.mode), + "Release", ], check=True, capture_output=True, @@ -230,7 +246,7 @@ def build_qt(self, options: dict): def build_boost(self, _=None): """ Builds boost shared libraries and installs libraries and headers """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "include", "boost")): + if os.path.exists(os.path.join(self.install_dir, "include", "boost-1_83")): print(" Not rebuilding boost, it is already in the LibPack") return # Boost uses a custom build system and needs a config file to find our Python @@ -249,26 +265,24 @@ def build_boost(self, _=None): subprocess.run([self.init_script, "&", "bootstrap.bat"], capture_output=True, check=True) subprocess.run([self.init_script, "&", "b2", f"install", - f"variant={str(self.mode).lower()}", "address-model=64", - "link=shared", f"--prefix=${self.install_dir}", - "--layout=versioned", # or system - "--build-type=complete"], # or minimal + "--layout=versioned", + "--build-type=complete", + f"stage"], check=True, capture_output=True) - #shutil.copytree(os.path.join("stage", "lib"), os.path.join(self.install_dir, "lib"), dirs_exist_ok=True) - #shutil.copytree("boost", os.path.join(self.install_dir, "include", "boost"), - # dirs_exist_ok=True) except subprocess.CalledProcessError as e: print("Error: failed to build boost") - print(e.output.decode("utf-8")) + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(e.returncode) - def _cmake_create_build_dir(self): - build_dir = "build-" + str(self.mode).lower() - if not os.path.exists(build_dir): - os.mkdir(build_dir) + def _cmake_create_build_dir(self, variant:BuildMode = BuildMode.RELEASE): + build_dir = "build-" + str(variant).lower() + if os.path.exists(build_dir): + shutil.rmtree(build_dir, onerror=remove_readonly) + os.mkdir(build_dir) os.chdir(build_dir) def _run_cmake(self, args): @@ -278,7 +292,9 @@ def _run_cmake(self, args): subprocess.run(cmake_setup_options, check=True, capture_output=True) except subprocess.CalledProcessError as e: print("ERROR: cMake failed!") - print(e.output.decode("utf-8")) + print (f"Command: {' '.join(cmake_setup_options)}") + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(e.returncode) def _cmake_configure(self, extra_args: list[str] = None): @@ -288,20 +304,26 @@ def _cmake_configure(self, extra_args: list[str] = None): options.append("..") self._run_cmake(options) - def _cmake_build(self): + def _cmake_build(self, variant:BuildMode = BuildMode.RELEASE): # Not building with --parallel because even with 32gb of RAM, OpenCASCADE runs out of memory on my system - cmake_build_options = ["--build", ".", "--config", str(self.mode)] + cmake_build_options = ["--build", ".", "--config", str(variant)] self._run_cmake(cmake_build_options) - def _cmake_install(self): - cmake_install_options = ["--install", "."] + def _cmake_install(self, variant:BuildMode = BuildMode.RELEASE): + cmake_install_options = ["--install", ".", "--config", str(variant)] self._run_cmake(cmake_install_options) def _build_standard_cmake(self, extra_args: list[str] = None): - self._cmake_create_build_dir() + self._cmake_create_build_dir(BuildMode.RELEASE) self._cmake_configure(extra_args) - self._cmake_build() - self._cmake_install() + self._cmake_build(BuildMode.RELEASE) + self._cmake_install(BuildMode.RELEASE) + os.chdir("..") + self._cmake_create_build_dir(BuildMode.DEBUG) + self._cmake_configure(extra_args) + self._cmake_build(BuildMode.DEBUG) + self._cmake_install(BuildMode.DEBUG) + os.chdir("..") def _pip_install(self, requirement: str) -> None: path_to_python = os.path.join(self.install_dir, "bin", "python") @@ -317,31 +339,11 @@ def _pip_install(self, requirement: str) -> None: def _build_with_pip(self, options: dict): if "pip-install" not in options: - print(f"ERROR: No pip-install provided in configuration of {options['name']}, so version cannot be determined") + print( + f"ERROR: No pip-install provided in configuration of {options['name']}, so version cannot be determined") exit(1) self._pip_install(options["pip-install"]) - def build_numpy(self, options=None): - self._build_with_pip(options) - - def build_scipy(self, options=None): - self._build_with_pip(options) - - def build_pillow(self, options=None): - self._build_with_pip(options) - - def build_pyyaml(self, options=None): - self._build_with_pip(options) - - def build_pycollada(self, options=None): - self._build_with_pip(options) - - def build_matplotlib(self, options=None): - self._build_with_pip(options) - - def build_opencv(self, options=None): - self._build_with_pip(options) - def build_coin(self, _=None): """ Builds and installs Coin using standard CMake settings """ if self.skip_existing: @@ -420,15 +422,32 @@ def build_libclang(self, _=None): shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True) def build_pyside(self, options=None): - """ As of Qt6, pyside is installed using pip """ + # Don't use a pip-install for this, we need the linkable libraries and include files for both PySide and + # Shiboken, which won't get installed by pip if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")): print(" Not rebuilding PySide6, it is already in the LibPack") return - if "pip-install" not in options: - print("ERROR: No pip-install provided in configuration of pyside, so version cannot be determined") + python = os.path.join(self.install_dir, "bin", "python") + qtpaths = "--qtpaths=" + os.path.join(self.install_dir, "bin", "qtpaths6") + ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") + clang = "CLANG_INSTALL_DIR=" + os.path.join(self.install_dir, "lib", "clang") + numpy = "--enable-numpy-support" + if sys.platform.startswith("win32"): + python += ".exe" + qtpaths += ".exe" + ssl += ".dll" + args = [self.init_script, "&", "set", clang, "&", python, "setup.py", "build", qtpaths, ssl, numpy] + else: + ssl += ".so" + args = [clang, "&&", python, "setup.py", "install", qtpaths, ssl, numpy] + try: + subprocess.run(args, capture_output=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to build Pyside and/or Shiboken") + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(1) - self._pip_install(options["pip-install"]) def build_vtk(self, _=None): if self.skip_existing: @@ -482,22 +501,31 @@ def build_tcl(self, _=None): if sys.platform.startswith("win32"): try: os.chdir("win") - args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] - subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, - "&", - "nmake", - "/f", - "makefile.vc", - "install", - f"INSTALLDIR={self.install_dir}"] - subprocess.run(args, check=True, capture_output=True) - self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"]) - self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"]) - self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"]) + for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] + if variant == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, + "&", + "nmake", + "/f", + "makefile.vc", + "install", + f"INSTALLDIR={self.install_dir}"] + subprocess.run(args, check=True, capture_output=True) + if variant == BuildMode.RELEASE: + self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"]) # Plus some debug? TODO what are they? + self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"]) + self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"]) + else: + self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh_d.exe"]) # Plus some debug? TODO what are they? + self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86_d.dll"]) + self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86_d.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tcl using nmake") - print(e.output.decode("utf-8")) + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(1) else: raise NotImplemented( @@ -513,14 +541,22 @@ def build_tk(self, _=None): if sys.platform.startswith("win32"): try: os.chdir("win") - args = [self.init_script, "&", "nmake", "/f", "makefile.vc", str(self.mode).lower()] - subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", - f"INSTALLDIR={self.install_dir}"] - subprocess.run(args, check=True, capture_output=True) - self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"]) - self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"]) - self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"]) + for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] + if variant == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", + f"INSTALLDIR={self.install_dir}"] + subprocess.run(args, check=True, capture_output=True) + if variant == BuildMode.RELEASE: + self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"]) # Plus some debug? TODO what are they? + self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"]) + self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"]) + else: + self.force_copy(["bin", "wish86t.exe"], ["bin", "wish_d.exe"]) # Plus some debug? TODO what are they? + self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86_d.dll"]) + self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86_d.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tk using nmake") print(e.output.decode("utf-8")) @@ -530,18 +566,40 @@ def build_tk(self, _=None): "Non-Windows compilation of tk is not implemented yet" ) + def build_rapidjson(self, _): + if os.path.exists(os.path.join(self.install_dir, "include", "rapidjson")): + if self.skip_existing: + print(" Not re-copying RapidJSON, it is already in the LibPack") + return + shutil.rmtree(os.path.join(self.install_dir, "include", "rapidjson")) + shutil.copytree("include", os.path.join(self.install_dir, "include"), dirs_exist_ok=True) + def build_opencascade(self, _=None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): print(" Not rebuilding OpenCASCADE, it is already in the LibPack") return - extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", - f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 - f"-D USE_VTK=On", - f"-D BUILD_CPP_STANDARD=C++17"] - if self.mode == BuildMode.DEBUG: - extra_args.append("-DBUILD_SHARED_LIBRARY_NAME_POSTFIX=d") - self._build_standard_cmake(extra_args=extra_args) + for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: + extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", + f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 + f"-D USE_VTK=On", + f"-D USE_RAPIDJSON=On", + f"-D BUILD_CPP_STANDARD=C++17", + f"-DBUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", + f"-DINSTALL_DIR_BIN=bin", + f"-DINSTALL_DIR_LIB=lib"] + if variant == BuildMode.DEBUG: + extra_args.append("-DBUILD_SHARED_LIBRARY_NAME_POSTFIX=d") + cwd = os.getcwd() + self._cmake_create_build_dir(variant) + self._cmake_configure(extra_args) + self._cmake_build(variant) + if variant == BuildMode.DEBUG and sys.platform.startswith("win32"): + # On Windows OpenCASCADE is looking in the wrong location for these files (as of 7.7.1) -- just copy them + # TODO - Don't hardcode the path + shutil.copytree(os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin")) + self._cmake_install() + os.chdir(cwd) def build_netgen(self, _: None): if self.skip_existing: @@ -551,14 +609,13 @@ def build_netgen(self, _: None): extra_args = [f"-DCMAKE_FIND_ROOT_PATH={self.install_dir}", "-D USE_SUPERBUILD=OFF", "-D USE_GUI=OFF", + "-D USE_NATIVE_ARCH=OFF", "-D USE_INTERNAL_TCL=OFF", f"-D TCL_DIR={self.install_dir}", f"-D TK_DIR={self.install_dir}", "-D USE_OCC=On", f"-D OpenCASCADE_ROOT={self.install_dir}", - f"-D USE_PYTHON=On", - "-D PYTHON_EXECUTABLE=" - f"{self.install_dir}/bin/python{'.exe' if sys.platform.startswith('win32') else ''}"] + f"-D USE_PYTHON=OFF"] self._build_standard_cmake(extra_args=extra_args) def build_hdf5(self, _: None): @@ -619,13 +676,14 @@ def build_icu(self, _: None): os.chdir(os.path.join("icu4c", "source")) if sys.platform.startswith("win32"): os.chdir("allinone") - args = [self.init_script, "&", "msbuild", f"/p:Configuration={self.mode}", - "/t:Build", "/p:SkipUWP=true", "allinone.sln"] - subprocess.run(args, check=True, capture_output=True) + for variant in ["Release", "Debug"]: + args = [self.init_script, "&", "msbuild", f"/p:Configuration={variant}", + "/t:Build", "/p:SkipUWP=true", "allinone.sln"] + subprocess.run(args, check=True, capture_output=True) + os.chdir(os.path.join("..", "..")) bin_dir = os.path.join(self.install_dir, "bin") lib_dir = os.path.join(self.install_dir, "lib") inc_dir = os.path.join(self.install_dir, "include") - os.chdir(os.path.join("..", "..")) shutil.copytree(f"bin64", bin_dir, dirs_exist_ok=True) shutil.copytree(f"lib64", lib_dir, dirs_exist_ok=True) shutil.copytree(f"include", inc_dir, dirs_exist_ok=True) @@ -664,4 +722,4 @@ def build_yamlcpp(self, _: None): print(" Not rebuilding yaml-cpp, it is already in the LibPack") return extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"] - self._build_standard_cmake((extra_args)) \ No newline at end of file + self._build_standard_cmake((extra_args)) diff --git a/config.json b/config.json index 4fb9494..032681a 100644 --- a/config.json +++ b/config.json @@ -34,6 +34,18 @@ "name":"opencv", "pip-install":"opencv-python==4.8.0.76" }, + { + "name":"cmake", + "pip-install":"cmake==3.27.2" + }, + { + "name":"setuptools", + "pip-install":"setuptools==68.1.2" + }, + { + "name":"wheel", + "pip-install":"wheel==0.41.2" + }, { "name":"zlib", "git-repo":"https://github.com/madler/zlib", @@ -86,7 +98,8 @@ }, { "name":"pyside", - "pip-install":"pyside6==6.5.2" + "git-repo": "http://code.qt.io/pyside/pyside-setup", + "git-ref": "v6.5.2" }, { "name":"vtk", @@ -113,6 +126,11 @@ "git-repo":"https://github.com/tcltk/tk", "git-ref":"core-8-6-13" }, + { + "name": "rapidjson", + "git-repo":"https://github.com/Tencent/rapidjson", + "git-ref":"v1.1.0" + }, { "name":"opencascade", "git-repo":"https://gitlab.com/blobfish/occt", diff --git a/create_libpack.py b/create_libpack.py index 8bb1921..c86f0d9 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -142,11 +142,11 @@ def decompress(name: str, filename: str): os.chdir(original_dir) -def write_manifest(outer_config: dict, mode): - manifest_file = os.path.join(compile_all.libpack_dir(outer_config, mode), "manifest.json") +def write_manifest(outer_config: dict): + manifest_file = os.path.join(compile_all.libpack_dir(outer_config), "manifest.json") with open(manifest_file, "w", encoding="utf-8") as f: f.write(json.dumps(outer_config["content"])) - version_file = os.path.join(compile_all.libpack_dir(outer_config, mode), "FREECAD_LIBPACK_VERSION") + version_file = os.path.join(compile_all.libpack_dir(outer_config), "FREECAD_LIBPACK_VERSION") with open(version_file, "w", encoding="utf-8") as f: f.write(outer_config["LibPack-version"]) @@ -169,15 +169,15 @@ def write_manifest(outer_config: dict, mode): ) parser.add_argument( "-e", - "--skip-existing-clone", - action="store_true", - help="If a given clone (or download) directory exists, skip cloning/downloading", + "--no-skip-existing-clone", + action="store_false", + help="If a given clone (or download) directory exists, delete it and download it again", ) parser.add_argument( "-b", - "--skip-existing-build", - action="store_true", - help="If a given build directory exists, skip building", + "--no-skip-existing-build", + action="store_false", + help="If a given build already exists, run the build process again anyway", ) parser.add_argument( "-s", @@ -196,7 +196,7 @@ def write_manifest(outer_config: dict, mode): path_to_7zip = args["7zip"] path_to_bison = args["bison"] - base = compile_all.libpack_dir(config, compile_all.BuildMode.RELEASE) + base = compile_all.libpack_dir(config) expected_py = os.path.join(base, "bin", "python") if sys.platform.startswith("win32"): expected_py += ".exe" @@ -206,16 +206,17 @@ def write_manifest(outer_config: dict, mode): exit(1) os.chdir(args["working"]) - fetch_remote_data(config, args["skip_existing_clone"]) + fetch_remote_data(config, args["no_skip_existing_clone"]) compiler = compile_all.Compiler( config, - compile_all.BuildMode.RELEASE, bison_path=path_to_bison, - skip_existing=args["skip_existing_build"], + skip_existing=args["no_skip_existing_build"], ) compiler.init_script = devel_init_script compiler.compile_all() - write_manifest(config, compile_all.BuildMode.RELEASE) + + + write_manifest(config) From 1b3d982b3188724c713cad10c4b5dff8633fef32 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sat, 27 Jan 2024 03:53:48 -0600 Subject: [PATCH 19/27] More attempts to get everything working --- bootstrap.py | 74 -------- compile_all.py | 434 +++++++++++++++++++++++++++++++--------------- config.json | 29 ++-- create_libpack.py | 79 ++++++--- test_bootstrap.py | 72 -------- 5 files changed, 368 insertions(+), 320 deletions(-) delete mode 100644 bootstrap.py delete mode 100644 test_bootstrap.py diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100644 index 3510777..0000000 --- a/bootstrap.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/python3 - -# SPDX-License-Identifier: LGPL-2.1-or-later - -import json -import os -import sys - -""" """ - -def parse_args() -> dict: - if len(sys.argv) > 3: - usage() - exit(1) - new_config_dict = {"config_file": "config.json"} - for arg in sys.argv[1:]: - key, value = extract_arg(arg) - new_config_dict[key] = value - return new_config_dict - - -def extract_arg(arg) -> tuple[str, object]: - return "config_file", arg - - -def usage(): - print("Used to create the base LibPack directory that you will then manually install Python into") - print("Usage: python bootstrap.py [config_file]") - print() - print('Result: A new working/LibPack-XX-YY directory has been created') - -def print_next_step(): - print('Next step: install Python into working/LibPack-XX-YY/bin, then run:') - print(' .\\working\\LibPack-XX-YY\\bin\\python create_libpack.py') - print('(where XX, YY change according to the config)') - - -def create_libpack_dir(config: dict) -> str: - """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of - the LibPack. Returns the name of the created directory. - """ - - dirname = "LibPack-{}-v{}".format( - config["FreeCAD-version"], - config["LibPack-version"] - ) - if os.path.exists(dirname): - backup_name = dirname + "-backup-" + "a" - while os.path.exists(backup_name): - if backup_name[-1] == "z": - print( - "You have too many old LibPack backup directories. Please delete some of them." - ) - exit(1) - backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) - - os.rename(dirname, backup_name) - if not os.path.exists(dirname): - os.mkdir(dirname) - dirname = os.path.join(dirname, "bin") - if not os.path.exists(dirname): - os.mkdir(dirname) - return dirname - - -if __name__ == "__main__": - args = parse_args() - with open(args["config_file"], "r", encoding="utf-8") as f: - config_data = f.read() - config_dict = json.loads(config_data) - os.makedirs("working", exist_ok=True) - os.chdir("working") - create_libpack_dir(config_dict) - print_next_step() \ No newline at end of file diff --git a/compile_all.py b/compile_all.py index 902433c..b8d41f9 100644 --- a/compile_all.py +++ b/compile_all.py @@ -6,6 +6,8 @@ # build script for each one. from diff_match_patch import diff_match_patch +from typing import Dict, List + from enum import Enum import os import pathlib @@ -51,7 +53,7 @@ def patch_single_file(filename, patch_data) -> None: f.write(new_text) -def split_patch_data(patch_data: str) -> list[dict[str, str]]: +def split_patch_data(patch_data: str) -> List[Dict[str, str]]: filename_regex = re.compile("@@@ ([^@]*) @@@\n") split_data = re.split(filename_regex, patch_data) result = [] @@ -79,7 +81,7 @@ def apply_patch(patch_file_path: str) -> None: patch_single_file(patch["file"], patch["data"]) -def patch_files(patches: list[str]) -> None: +def patch_files(patches: List[str]) -> None: """ Given a list of patches, apply them sequentially in the current working directory. The patches themselves are expected to be given as paths relative to **this** Python script file""" for patch in patches: @@ -88,26 +90,51 @@ def patch_files(patches: list[str]) -> None: apply_patch(patch) -def libpack_dir(config: dict): - lp_dir = "LibPack-{}-v{}".format( +def libpack_dir(config: dict, mode: BuildMode): + lp_dir = "LibPack-{}-v{}-{}".format( config["FreeCAD-version"], config["LibPack-version"], + str(mode) ) return os.path.join(os.path.dirname(__file__), "working", lp_dir) +def to_exe(base: str = ""): + """ Append .exe to Windows executables, but not to macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to an executable's name. """ + return base + '.exe' if sys.platform.startswith('win32') else '' + + +def to_static(base: str = ""): + """ Append .lib to Windows libraries, or .a macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to a static library's name. """ + return base + '.lib' if sys.platform.startswith('win32') else '.a' + + +def to_dynamic(base: str = ""): + """ Append .dll to Windows libraries, or .so to macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to a dynamic library's name. """ + return base + '.dll' if sys.platform.startswith('win32') else '.so' + + class Compiler: - def __init__(self, config, bison_path, skip_existing: bool = False): + def __init__(self, config, bison_path, skip_existing: bool = False, mode: BuildMode = BuildMode.RELEASE): self.config = config self.bison_path = bison_path self.base_dir = os.getcwd() self.skip_existing = skip_existing - self.install_dir = libpack_dir(config) + self.install_dir = libpack_dir(config, mode) self.init_script = None + self.mode = mode - def get_cmake_options(self) -> list[str]: + def get_cmake_options(self) -> List[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ + pcre_lib = self.install_dir + "/lib/pcre2-8" + if self.mode == BuildMode.DEBUG: + pcre_lib += "d" + pcre_lib += to_static() + base = [ f'-D BISON_EXECUTABLE={self.bison_path}', f'-D BOOST_ROOT={self.install_dir}', @@ -121,8 +148,9 @@ def get_cmake_options(self) -> list[str]: f'-D BUILD_TESTS=No', f'-D BUILD_TESTING=No', f'-D BZIP2_DIR={self.install_dir}/lib/cmake/', - f'-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/', + f'-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/', # TODO Remove hardcoded version f'-D Boost_INCLUDE_DIRS={self.install_dir}/include', + f'-D CMAKE_BUILD_TYPE={self.mode}', f'-D CMAKE_INSTALL_PATH={self.install_dir}', f'-D CMAKE_INSTALL_PREFIX={self.install_dir}', f'-D Coin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1', @@ -130,21 +158,20 @@ def get_cmake_options(self) -> list[str]: f'-D HDF5_DIR={self.install_dir}/share/cmake/', f'-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib', f'-D HDF5_LIBRARY_RELEASE={self.install_dir}/lib/hdf5.lib', - f'-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff' + '.exe' if sys.platform.startswith( - 'win32') else '', + f'-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff' + to_exe(), f'-D INSTALL_DIR={self.install_dir}', - f'-D PCRE2_LIBRARY={self.install_dir}/lib/pcre2-8.lib', + f'-D PCRE2_LIBRARY={pcre_lib}', f'-D Python_ROOT_DIR={self.install_dir}/bin', + f'-D Python_DIR={self.install_dir}/bin', + '-D Python_FIND_REGISTRY=NEVER', f'-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6', - f'-D SWIG_EXECUTABLE={self.install_dir}/bin/swig' + '.exe' if sys.platform.startswith('win32') else '', + f'-D SWIG_EXECUTABLE={self.install_dir}/bin/swig' + to_exe(), f'-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO', # Workaround for bug in Visual Studio MSVC 143 f'-D VTK_MODULE_ENABLE_VTK_ioss=NO', # Workaround for bug in Visual Studio MSVC 143 f'-D ZLIB_DIR={self.install_dir}/lib/cmake/', f'-D ZLIB_INCLUDE_DIR={self.install_dir}/include', - f'-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib.' + 'lib' if sys.platform.startswith( - 'win32') else 'a', - f'-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd.' + 'lib' if sys.platform.startswith( - 'win32') else 'a', + f'-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib' + to_static(), + f'-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd' + to_static(), ] if sys.platform.startswith('win32'): inc_path = self.install_dir.replace('\\', '/') @@ -175,10 +202,18 @@ def compile_all(self): def build_nonexistent(self, _=None): """Used for automated testing to allow easy Mock injection""" + def python_exe(self): + if self.mode == BuildMode.RELEASE: + return os.path.join(self.install_dir, "bin", "python") + to_exe() + return os.path.join(self.install_dir, "bin", "python_d") + to_exe() + def build_python(self, _=None): - """ NOTE: This doesn't install correctly, so should not be used at this time... install Python manually """ + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "bin", "DLLs")): + print(" Not rebuilding Python, it is already in the LibPack") + return if sys.platform.startswith("win32"): - expected_exe_path = os.path.join(self.install_dir, "bin", "python.exe") + expected_exe_path = self.python_exe() if self.skip_existing and os.path.exists(expected_exe_path): print("Not rebuilding, instead just using existing Python in the LibPack installation path") return @@ -192,33 +227,107 @@ def build_python(self, _=None): "-p", arch, "-c", - "Release", + str(self.mode), ], check=True, capture_output=True, ) except subprocess.CalledProcessError as e: print("Python build failed") - print(e.output.decode("utf-8")) + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(e.returncode) bin_dir = os.path.join(self.install_dir, "bin") + dll_dir = os.path.join(bin_dir, "DLLs") lib_dir = os.path.join(bin_dir, "Lib") + libs_dir = os.path.join(bin_dir, "libs") inc_dir = os.path.join(bin_dir, "Include") + tools_dir = os.path.join(bin_dir, "Tools") os.makedirs(bin_dir, exist_ok=True) + os.makedirs(dll_dir, exist_ok=True) os.makedirs(lib_dir, exist_ok=True) + os.makedirs(libs_dir, exist_ok=True) os.makedirs(bin_dir, exist_ok=True) - shutil.copytree(f"PCBuild\\{path}", bin_dir, dirs_exist_ok=True) + os.makedirs(tools_dir, exist_ok=True) + tools_subs = ["i18n", "scripts", "demo"] + for sub in tools_subs: + os.makedirs(os.path.join(tools_dir, sub), exist_ok=True) + + # NOTES: + # When installed via the Python installer, the top-level Python folder contains: + # python.exe + # python.pdb + # python3.dll + # python311.dll + # python311.pdb + # python311_d.dll + # python311_d.pdb + # python3_d.dll + # pythonw.exe + # pythonw.pdb + # pythonw_d.exe + # pythonw_d.pdb + # python_d.exe + # python_d.pdb + # vcruntime140.dll + # vcruntime140_1.dll + # It also contains 5 subdirectories: DLLs, include, Lib, libs, and Tools, plus LICENSE.txt + # DLLS folder contains *.pyd, *.pdb, and *.dll + # include contains the header file directory tree + # Lib contains the Python standard libraries + # libs contains the actual Python *.lib files (python3.lib and python311.lib and their debug equivalents + # Tools contains a number of subdirectories with Python scripts: i18n, scripts, and demo + # Finally, we also need the file "pyconfig.h" which is in yet another directory of the Python build, "PC" + + shutil.copytree(f"PCBuild\\{path}", dll_dir, dirs_exist_ok=True) shutil.copytree(f"Lib", lib_dir, dirs_exist_ok=True) shutil.copytree(f"Include", inc_dir, dirs_exist_ok=True) + for sub in tools_subs: + shutil.copytree(f"Tools\\{sub}", os.path.join(tools_dir, sub), dirs_exist_ok=True) + + # Figure out what version of Python we just built: + major, minor = self.get_python_version(os.path.join("PCBuild", path, "python.exe")).split(".") + + # Construct the list of files we expect to exist that need to be placed in the toplevel directory, or in + # libs: + move_to_bin = ["vcruntime"] + for base in ["python", f"python{major}", f"python{major}{minor}", "pythonw"]: + final = base + if self.mode == BuildMode.DEBUG: + final += "_d" + move_to_bin.append(final) + # They are all in the DLLs subdirectory now: move the ones that match: + for file in pathlib.Path(dll_dir).iterdir(): + if file.is_file(): + if file.stem in move_to_bin: + if file.suffix == ".lib": + target = os.path.join(libs_dir, file.name) + elif file.suffix in [".dll", ".exe", ".pdb"]: + target = os.path.join(bin_dir, file.name) + else: + continue + if os.path.exists(target): + os.unlink(target) + file.rename(target) + # Make a link from python_d.exe to python.exe: mklink /h python.exe python_d.exe (for exes and libs) + pyconfig = os.path.join("PC", "pyconfig.h") + target = os.path.join(inc_dir, "pyconfig.h") + if not os.path.exists(pyconfig): + print("ERROR: Could not locate pyconfig.h, cannot complete installation of Python") + exit(1) + if os.path.exists(target): + os.unlink(target) + os.rename(pyconfig, target) else: raise NotImplemented( "Non-Windows compilation of Python is not implemented yet" ) - def get_python_version(self) -> str: - path_to_python = os.path.join(self.install_dir, "bin", "python") - if sys.platform.startswith("win32"): - path_to_python += ".exe" + def get_python_version(self, exe: str = None) -> str: + if exe is None: + path_to_python = self.python_exe() + else: + path_to_python = exe try: result = subprocess.run([path_to_python, "--version"], capture_output=True, check=True) _, _, version_number = result.stdout.decode("utf-8").strip().partition(" ") @@ -227,7 +336,18 @@ def get_python_version(self) -> str: return python_version except subprocess.CalledProcessError as e: print("ERROR: Failed to run LibPack's Python executable") - print(e.output.decode("utf-8")) + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) + exit(1) + + def build_pip(self, _=None): + path_to_python = self.python_exe() + try: + subprocess.run([path_to_python, "-m", "ensurepip"], capture_output=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: Failed to run LibPack's Python executable") + print(e.stdout.decode("utf-8")) + print(e.stderr.decode("utf-8")) exit(1) def build_qt(self, options: dict): @@ -251,35 +371,53 @@ def build_boost(self, _=None): return # Boost uses a custom build system and needs a config file to find our Python with open(os.path.join("tools", "build", "src", "user-config.jam"), "w", encoding="utf-8") as user_config: - exe = os.path.join(self.install_dir, "bin", "python") + exe = self.python_exe() if sys.platform.startswith("win32"): - exe += ".exe" exe = exe.replace("\\", "\\\\") inc_dir = os.path.join(self.install_dir, "bin", "include").replace("\\", "\\\\") lib_dir = os.path.join(self.install_dir, "bin", "libs").replace("\\", "\\\\") python_version = self.get_python_version() - print(f" (boost-python is being built against Python {python_version})") - user_config.write(f'using python : {python_version} : "{exe}" : "{inc_dir}" : "{lib_dir}" ;\n') + full_version = python_version + "d" if self.mode == BuildMode.DEBUG else "" + print(f" (boost-python is being built against Python {full_version})") + user_config.write(f'using python : {python_version} ') + user_config.write(f': "{exe}" ') + user_config.write(f': "{inc_dir}" ') + user_config.write(f': "{lib_dir}" ') + if self.mode == BuildMode.DEBUG: + user_config.write(f': on ') + user_config.write(";\n") try: # When debugging on the command line, add --debug-configuration to get more verbose output subprocess.run([self.init_script, "&", "bootstrap.bat"], capture_output=True, check=True) subprocess.run([self.init_script, "&", "b2", f"install", "address-model=64", + "link=static,shared", + str(self.mode).lower(), + "python-debugging=" + "on" if self.mode == BuildMode.DEBUG else "off", f"--prefix=${self.install_dir}", "--layout=versioned", "--build-type=complete", f"stage"], check=True, capture_output=True) + # NOTE: I get an error when running this here, but when I run what I believe is the same command from a + # command line, it works fine. except subprocess.CalledProcessError as e: - print("Error: failed to build boost") - print(e.stdout.decode("utf-8")) - print(e.stderr.decode("utf-8")) + # Boost is too verbose in its output to be of much use un-processed. Dump it all to a file, and + # then print only the lines with the word "error" on them to stdout + print("Error: failed to build boost -- writing output to stdout.txt") + with open("stdout.txt", "w", encoding="utf-8") as f: + f.write(e.stdout.decode("utf-8")) + lines = e.stdout.decode("utf-8").split("\n") + for line in lines: + if "error" in line.lower(): + # Lots of these lines are just files with the word 'error' in them, maybe there is a better filter? + print(line) exit(e.returncode) - def _cmake_create_build_dir(self, variant:BuildMode = BuildMode.RELEASE): - build_dir = "build-" + str(variant).lower() + def _cmake_create_build_dir(self): + build_dir = "build-" + str(self.mode).lower() if os.path.exists(build_dir): shutil.rmtree(build_dir, onerror=remove_readonly) os.mkdir(build_dir) @@ -292,43 +430,36 @@ def _run_cmake(self, args): subprocess.run(cmake_setup_options, check=True, capture_output=True) except subprocess.CalledProcessError as e: print("ERROR: cMake failed!") - print (f"Command: {' '.join(cmake_setup_options)}") + print(f"Command: {' '.join(cmake_setup_options)}") print(e.stdout.decode("utf-8")) print(e.stderr.decode("utf-8")) exit(e.returncode) - def _cmake_configure(self, extra_args: list[str] = None): + def _cmake_configure(self, extra_args: List[str] = None): options = self.get_cmake_options() if extra_args: options.extend(extra_args) options.append("..") self._run_cmake(options) - def _cmake_build(self, variant:BuildMode = BuildMode.RELEASE): - # Not building with --parallel because even with 32gb of RAM, OpenCASCADE runs out of memory on my system - cmake_build_options = ["--build", ".", "--config", str(variant)] + def _cmake_build(self, parallel: bool = True): + cmake_build_options = ["--build", ".", "--config", str(self.mode).lower()] + if parallel: + cmake_build_options.append("--parallel") self._run_cmake(cmake_build_options) - def _cmake_install(self, variant:BuildMode = BuildMode.RELEASE): - cmake_install_options = ["--install", ".", "--config", str(variant)] + def _cmake_install(self): + cmake_install_options = ["--install", ".", "--config", str(self.mode).lower()] self._run_cmake(cmake_install_options) - def _build_standard_cmake(self, extra_args: list[str] = None): - self._cmake_create_build_dir(BuildMode.RELEASE) - self._cmake_configure(extra_args) - self._cmake_build(BuildMode.RELEASE) - self._cmake_install(BuildMode.RELEASE) - os.chdir("..") - self._cmake_create_build_dir(BuildMode.DEBUG) + def _build_standard_cmake(self, extra_args: List[str] = None): + self._cmake_create_build_dir() self._cmake_configure(extra_args) - self._cmake_build(BuildMode.DEBUG) - self._cmake_install(BuildMode.DEBUG) - os.chdir("..") + self._cmake_build() + self._cmake_install() def _pip_install(self, requirement: str) -> None: - path_to_python = os.path.join(self.install_dir, "bin", "python") - if sys.platform.startswith("win32"): - path_to_python += ".exe" + path_to_python = self.python_exe() try: subprocess.run([path_to_python, "-m", "pip", "install", requirement], check=True, capture_output=True) @@ -339,8 +470,7 @@ def _pip_install(self, requirement: str) -> None: def _build_with_pip(self, options: dict): if "pip-install" not in options: - print( - f"ERROR: No pip-install provided in configuration of {options['name']}, so version cannot be determined") + print(f"ERROR: No pip-install provided in config of {options['name']}, so version cannot be determined") exit(1) self._pip_install(options["pip-install"]) @@ -399,8 +529,7 @@ def build_pcre2(self, _=None): def build_swig(self, _=None): if self.skip_existing: - if os.path.exists( - os.path.join(self.install_dir, "bin", "swig") + ".exe" if sys.platform.startswith("win32") else ""): + if os.path.exists(os.path.join(self.install_dir, "bin", "swig") + to_exe()): print(" Not rebuilding SWIG, it is already in the LibPack") return self._build_standard_cmake() @@ -411,6 +540,10 @@ def build_pivy(self, _=None): print(" Not rebuilding pivy, it is already in the LibPack") return self._build_standard_cmake() + if self.mode == BuildMode.DEBUG: + base = os.path.join(self.install_dir,"bin","Lib","site-packages","pivy") + os.rename(os.path.join(base, "_coin.pyd"), os.path.join(base, "_coin_d.pyd")) + def build_libclang(self, _=None): """ libclang is provided as a platform-specific download by Qt. """ @@ -421,26 +554,28 @@ def build_libclang(self, _=None): print(" (not really building libclang, just copying from a build provided by Qt)") shutil.copytree("libclang", self.install_dir, dirs_exist_ok=True) - def build_pyside(self, options=None): + def build_pyside(self, _=None): # Don't use a pip-install for this, we need the linkable libraries and include files for both PySide and - # Shiboken, which won't get installed by pip + # Shiboken, which won't get installed by pip, and it needs to be built against the right Python exe if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")): print(" Not rebuilding PySide6, it is already in the LibPack") return - python = os.path.join(self.install_dir, "bin", "python") - qtpaths = "--qtpaths=" + os.path.join(self.install_dir, "bin", "qtpaths6") - ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") + python = self.python_exe() + qtpaths = "--qtpaths=" + os.path.join(self.install_dir, "bin", "qtpaths6") + to_exe() clang = "CLANG_INSTALL_DIR=" + os.path.join(self.install_dir, "lib", "clang") - numpy = "--enable-numpy-support" + vulkan = "VULKAN_SDK=None" #"VULKAN_SDK=" + os.path.join(self.install_dir, "Vulkan") + parallel = "--parallel=16" + # numpy = "--enable-numpy-support" if sys.platform.startswith("win32"): - python += ".exe" - qtpaths += ".exe" - ssl += ".dll" - args = [self.init_script, "&", "set", clang, "&", python, "setup.py", "build", qtpaths, ssl, numpy] + ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") + args = [self.init_script, "&", "set", clang, "&", "set", vulkan, "&", python, "setup.py", "install", + qtpaths, ssl, parallel] + if self.mode == BuildMode.DEBUG: + args.append ("--debug") else: - ssl += ".so" - args = [clang, "&&", python, "setup.py", "install", qtpaths, ssl, numpy] + ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") + args = [clang, "&&", python, "setup.py", "install", qtpaths, ssl] try: subprocess.run(args, capture_output=True, check=True) except subprocess.CalledProcessError as e: @@ -477,8 +612,12 @@ def build_freetype(self, _=None): print(" Not rebuilding freetype, it is already in the LibPack") return self._build_standard_cmake() + if self.mode == BuildMode.DEBUG: + # OCCT *really* wants these libraries named like this: + shutil.copyfile(f"{self.install_dir}/bin/freetyped.dll",f"{self.install_dir}/bin/freetype.dll") + shutil.copyfile(f"{self.install_dir}/lib/freetyped.lib",f"{self.install_dir}/lib/freetype.lib") - def force_copy(self, src_components: list[str], dst_components: list[str]): + def force_copy(self, src_components: List[str], dst_components: List[str]): full_src = self.install_dir for src in src_components: full_src = os.path.join(full_src, src) @@ -501,27 +640,28 @@ def build_tcl(self, _=None): if sys.platform.startswith("win32"): try: os.chdir("win") - for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: - args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] - if variant == BuildMode.DEBUG: - args.append("OPTS=symbols") - subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, - "&", - "nmake", - "/f", - "makefile.vc", - "install", - f"INSTALLDIR={self.install_dir}"] - subprocess.run(args, check=True, capture_output=True) - if variant == BuildMode.RELEASE: - self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"]) # Plus some debug? TODO what are they? - self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"]) - self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"]) - else: - self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh_d.exe"]) # Plus some debug? TODO what are they? - self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86_d.dll"]) - self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86_d.lib"]) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] + if self.mode == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, + "&", + "nmake", + "/f", + "makefile.vc", + "install", + f"INSTALLDIR={self.install_dir}"] + if self.mode == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + if self.mode == BuildMode.RELEASE: + self.force_copy(["bin", "tclsh86t.exe"], ["bin", "tclsh.exe"]) + self.force_copy(["bin", "tcl86t.dll"], ["bin", "tcl86.dll"]) + self.force_copy(["lib", "tcl86t.lib"], ["lib", "tcl86.lib"]) + else: + self.force_copy(["bin", "tclsh86tg.exe"], ["bin", "tclsh.exe"]) + self.force_copy(["bin", "tcl86tg.dll"], ["bin", "tcl86.dll"]) + self.force_copy(["lib", "tcl86tg.lib"], ["lib", "tcl86.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tcl using nmake") print(e.stdout.decode("utf-8")) @@ -541,22 +681,23 @@ def build_tk(self, _=None): if sys.platform.startswith("win32"): try: os.chdir("win") - for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: - args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] - if variant == BuildMode.DEBUG: - args.append("OPTS=symbols") - subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", - f"INSTALLDIR={self.install_dir}"] - subprocess.run(args, check=True, capture_output=True) - if variant == BuildMode.RELEASE: - self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"]) # Plus some debug? TODO what are they? - self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"]) - self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"]) - else: - self.force_copy(["bin", "wish86t.exe"], ["bin", "wish_d.exe"]) # Plus some debug? TODO what are they? - self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86_d.dll"]) - self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86_d.lib"]) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc", "release"] + if self.mode == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", + f"INSTALLDIR={self.install_dir}"] + if self.mode == BuildMode.DEBUG: + args.append("OPTS=symbols") + subprocess.run(args, check=True, capture_output=True) + if self.mode == BuildMode.RELEASE: + self.force_copy(["bin", "wish86t.exe"], ["bin", "wish.exe"]) + self.force_copy(["bin", "tk86t.dll"], ["bin", "tk86.dll"]) + self.force_copy(["lib", "tk86t.lib"], ["lib", "tk86.lib"]) + else: + self.force_copy(["bin", "wish86tg.exe"], ["bin", "wish.exe"]) + self.force_copy(["bin", "tk86tg.dll"], ["bin", "tk86.dll"]) + self.force_copy(["lib", "tk86tg.lib"], ["lib", "tk86.lib"]) except subprocess.CalledProcessError as e: print("ERROR: Failed to build tk using nmake") print(e.output.decode("utf-8")) @@ -579,34 +720,44 @@ def build_opencascade(self, _=None): if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): print(" Not rebuilding OpenCASCADE, it is already in the LibPack") return - for variant in [BuildMode.RELEASE, BuildMode.DEBUG]: - extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", - f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 - f"-D USE_VTK=On", - f"-D USE_RAPIDJSON=On", - f"-D BUILD_CPP_STANDARD=C++17", - f"-DBUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", - f"-DINSTALL_DIR_BIN=bin", - f"-DINSTALL_DIR_LIB=lib"] - if variant == BuildMode.DEBUG: - extra_args.append("-DBUILD_SHARED_LIBRARY_NAME_POSTFIX=d") - cwd = os.getcwd() - self._cmake_create_build_dir(variant) - self._cmake_configure(extra_args) - self._cmake_build(variant) - if variant == BuildMode.DEBUG and sys.platform.startswith("win32"): - # On Windows OpenCASCADE is looking in the wrong location for these files (as of 7.7.1) -- just copy them - # TODO - Don't hardcode the path - shutil.copytree(os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin")) - self._cmake_install() - os.chdir(cwd) + extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", + f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 + f"-D USE_VTK=On", + f"-D USE_RAPIDJSON=On", + f"-D BUILD_CPP_STANDARD=C++17", + f"-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", + f"-D INSTALL_DIR_BIN=bin", + f"-D INSTALL_DIR_LIB=lib"] + if self.mode == BuildMode.DEBUG: + extra_args.append("-D BUILD_SHARED_LIBRARY_NAME_POSTFIX=d") + cwd = os.getcwd() + self._cmake_create_build_dir() + self._cmake_configure(extra_args) + self._cmake_build(parallel=False) + if self.mode == BuildMode.DEBUG and sys.platform.startswith("win32"): + # On Windows OpenCASCADE is looking in the wrong location for these files (as of 7.7.1) -- just copy them + # TODO - Don't hardcode the path + shutil.copytree(os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin")) + self._cmake_install() + + os.chdir(cwd) + + # TODO - something is getting messed up in the CMake config output (note the quotes around 26812): for now just + # drop the line entirely + # set (OpenCASCADE_CXX_FLAGS "/IG:/FreeCAD/FreeCAD-LibPack-chennes/working/LibPack-0.22-v3.0.0-Debug/include /EHa /fp:precise /fp:precise /wd"26812" /MP /W4") + with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "r", encoding="utf-8") as f: + occt_cmake_contents = f.readlines() + with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "w", encoding="utf-8") as f: + for line in occt_cmake_contents: + if "OpenCASCADE_CXX_FLAGS" not in line: + f.write(line + "\n") def build_netgen(self, _: None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "share", "netgen")): print(" Not rebuilding netgen, it is already in the LibPack") return - extra_args = [f"-DCMAKE_FIND_ROOT_PATH={self.install_dir}", + extra_args = [f"-D CMAKE_FIND_ROOT_PATH={self.install_dir}", "-D USE_SUPERBUILD=OFF", "-D USE_GUI=OFF", "-D USE_NATIVE_ARCH=OFF", @@ -623,7 +774,7 @@ def build_hdf5(self, _: None): if os.path.exists(os.path.join(self.install_dir, "include", "hdf5.h")): print(" Not rebuilding hdf5, it is already in the LibPack") return - # Something goes wrong with the install during this script, but when run from the command line it succeeds + # Something goes wrong with the installation during this script, but when run from the command line it succeeds # without a problem. self._build_standard_cmake() @@ -632,15 +783,13 @@ def build_medfile(self, _: None): if os.path.exists(os.path.join(self.install_dir, "include", "medfile.h")): print(" Not rebuilding medfile, it is already in the LibPack") return - extra_args = ["-DMEDFILE_USE_UNICODE=On"] + extra_args = ["-D MEDFILE_USE_UNICODE=On"] self._build_standard_cmake(extra_args) def build_gmsh(self, _: None): if self.skip_existing: if os.path.exists( - os.path.join(self.install_dir, - "bin", - "gmsh" + ".exe" if sys.platform.startswith("win32") else "")): + os.path.join(self.install_dir, "bin", "gmsh" + to_exe())): print(" Not rebuilding gmsh, it is already in the LibPack") return extra_args = [] @@ -655,9 +804,7 @@ def build_pycxx(self, _: None): if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "CXX")): print(" Not rebuilding PyCXX, it is already in the LibPack") return - path_to_python = os.path.join(self.install_dir, "bin", "python") - if sys.platform.startswith("win32"): - path_to_python += ".exe" + path_to_python = self.python_exe() args = [path_to_python, "setup.py", "install"] try: subprocess.run(args, check=True, capture_output=True) @@ -676,10 +823,9 @@ def build_icu(self, _: None): os.chdir(os.path.join("icu4c", "source")) if sys.platform.startswith("win32"): os.chdir("allinone") - for variant in ["Release", "Debug"]: - args = [self.init_script, "&", "msbuild", f"/p:Configuration={variant}", - "/t:Build", "/p:SkipUWP=true", "allinone.sln"] - subprocess.run(args, check=True, capture_output=True) + args = [self.init_script, "&", "msbuild", f"/p:Configuration={str(self.mode).lower()}", + "/t:Build", "/p:SkipUWP=true", "allinone.sln"] + subprocess.run(args, check=True, capture_output=True) os.chdir(os.path.join("..", "..")) bin_dir = os.path.join(self.install_dir, "bin") lib_dir = os.path.join(self.install_dir, "lib") @@ -722,4 +868,4 @@ def build_yamlcpp(self, _: None): print(" Not rebuilding yaml-cpp, it is already in the LibPack") return extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"] - self._build_standard_cmake((extra_args)) + self._build_standard_cmake(extra_args) diff --git a/config.json b/config.json index 032681a..8c45bc1 100644 --- a/config.json +++ b/config.json @@ -3,8 +3,21 @@ "LibPack-version":"3.0.0", "content": [ { - "name":"qt", - "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" + "name":"python", + "git-repo":"https://github.com/python/cpython.git", + "git-ref":"v3.11.5" + }, + { + "name":"pip", + "note":"Use the ensure_pip Python module to install pip after compiling Python" + }, + { + "name":"setuptools", + "pip-install":"setuptools==68.2.0" + }, + { + "name":"wheel", + "pip-install":"wheel==0.41.2" }, { "name":"numpy", @@ -38,19 +51,15 @@ "name":"cmake", "pip-install":"cmake==3.27.2" }, - { - "name":"setuptools", - "pip-install":"setuptools==68.1.2" - }, - { - "name":"wheel", - "pip-install":"wheel==0.41.2" - }, { "name":"zlib", "git-repo":"https://github.com/madler/zlib", "git-ref":"v1.3" }, + { + "name":"qt", + "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" + }, { "name":"bzip2", "git-repo":"https://gitlab.com/bzip2/bzip2.git", diff --git a/create_libpack.py b/create_libpack.py index c86f0d9..ac64109 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -15,6 +15,16 @@ # * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF # * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) +# Note about Python: Python includes the following dependencies when built on Windows (as of v3.11.5) +# bzip2-1.0.8 +# sqlite-3.42.0.0 +# xz-5.2.5 +# zlib-1.2.13 +# libffi-3.4.4 +# openssl-bin-3.0.9 +# tcltk-8.6.12.1 +# At present these are not re-used + import argparse import json import os @@ -75,6 +85,29 @@ def load_config(path: str) -> dict: exit(1) +def create_libpack_dir(config: dict, mode: compile_all.BuildMode) -> str: + """Create a new directory for this LibPack compilation, using the version of FreeCAD, the version of + the LibPack, and whether it's in release or debug mode. Returns the name of the created directory. + """ + + dirname = compile_all.libpack_dir(config, mode) + if os.path.exists(dirname): + backup_name = dirname + "-backup-" + "a" + while os.path.exists(backup_name): + if backup_name[-1] == "z": + print("You have too many old LibPack backup directories. Please delete some of them.") + exit(1) + backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) + + os.rename(dirname, backup_name) + if not os.path.exists(dirname): + os.mkdir(dirname) + dirname = os.path.join(dirname, "bin") + if not os.path.exists(dirname): + os.mkdir(dirname) + return dirname + + def fetch_remote_data(config: dict, skip_existing: bool = False): """Clone the required repos and download the URLs""" content = config["content"] @@ -142,11 +175,11 @@ def decompress(name: str, filename: str): os.chdir(original_dir) -def write_manifest(outer_config: dict): - manifest_file = os.path.join(compile_all.libpack_dir(outer_config), "manifest.json") +def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): + manifest_file = os.path.join(compile_all.libpack_dir(outer_config, mode_used), "manifest.json") with open(manifest_file, "w", encoding="utf-8") as f: - f.write(json.dumps(outer_config["content"])) - version_file = os.path.join(compile_all.libpack_dir(outer_config), "FREECAD_LIBPACK_VERSION") + f.write(json.dumps(outer_config["content"], indent=" ")) + version_file = os.path.join(compile_all.libpack_dir(outer_config, mode_used), "FREECAD_LIBPACK_VERSION") with open(version_file, "w", encoding="utf-8") as f: f.write(outer_config["LibPack-version"]) @@ -155,6 +188,12 @@ def write_manifest(outer_config: dict): parser = argparse.ArgumentParser( description="Builds a collection of FreeCAD dependencies for the current system" ) + parser.add_argument( + "-m", + "--mode", + help="'release' or 'debug''", + default="release", + ) parser.add_argument( "-c", "--config", @@ -192,31 +231,31 @@ def write_manifest(outer_config: dict): parser.add_argument("path-to-final-libpack-dir", nargs="?", default="./") args = vars(parser.parse_args()) - config = load_config(args["config"]) + config_dict = load_config(args["config"]) path_to_7zip = args["7zip"] path_to_bison = args["bison"] - base = compile_all.libpack_dir(config) - expected_py = os.path.join(base, "bin", "python") - if sys.platform.startswith("win32"): - expected_py += ".exe" - if not os.path.exists(expected_py): - print(f"ERROR: Could not find Python at {expected_py}") - print("Run the bootstrap.py script and then install Python into the created 'bin' directory") - exit(1) - os.chdir(args["working"]) + os.makedirs("working", exist_ok=True) + os.chdir("working") + mode = compile_all.BuildMode.DEBUG if args["mode"].lower() == "debug" else compile_all.BuildMode.RELEASE + if args["no_skip_existing_clone"]: + dirname = compile_all.libpack_dir(config_dict, mode) + if not os.path.exists(dirname): + base = create_libpack_dir(config_dict, mode) + else: + base = dirname + else: + base = create_libpack_dir(config_dict, mode) - fetch_remote_data(config, args["no_skip_existing_clone"]) + fetch_remote_data(config_dict, args["no_skip_existing_clone"]) compiler = compile_all.Compiler( - config, + config_dict, bison_path=path_to_bison, skip_existing=args["no_skip_existing_build"], + mode=mode ) compiler.init_script = devel_init_script compiler.compile_all() - - - write_manifest(config) - + write_manifest(config_dict, mode) diff --git a/test_bootstrap.py b/test_bootstrap.py deleted file mode 100644 index 1b06674..0000000 --- a/test_bootstrap.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/python3 -import compile_all -# SPDX-License-Identifier: LGPL-2.1-or-later - -import os -import tempfile -import unittest - -import bootstrap - -""" Developer tests for the bootstrap module. """ - - -class TestBootstrap(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.original_dir = os.getcwd() - self.config = {"FreeCAD-version": "0.22", - "LibPack-version": "3.0.0", - "content": [ - {"name": "nonexistent"} - ]} - - def tearDown(self) -> None: - super().tearDown() - - def test_create_libpack_dir_no_conflict(self): - with tempfile.TemporaryDirectory() as temp_dir: - os.chdir(temp_dir) - try: - dirname = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) - self.assertNotEqual(dirname.find("0.22"), -1) - self.assertNotEqual(dirname.find("3.0.0"), -1) - self.assertNotEqual(dirname.find("Release"), -1) - except: - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - raise - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - - def test_create_libpack_dir_needs_backup(self): - with tempfile.TemporaryDirectory() as temp_dir: - os.chdir(temp_dir) - try: - first = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) - second = bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) - self.assertTrue(os.path.exists(second)) - self.assertEqual(len(os.listdir()), 2) - except: - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - raise - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - - def test_too_many_backups_exist(self): - with tempfile.TemporaryDirectory() as temp_dir: - try: - # Arrange - os.chdir(temp_dir) - config = {"FreeCAD-version": "0.22", "LibPack-version": "3.0.0"} - for i in range(0, 27): - bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) - self.assertEqual(len(os.listdir()), i + 1) - # Act - with self.assertRaises(SystemExit): - bootstrap.create_libpack_dir(self.config, compile_all.BuildMode.RELEASE) - except: - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - raise - os.chdir(self.original_dir) # Otherwise we can't unlink the directory on Windows - - -if __name__ == "__main__": - unittest.main() From 35948268ddc3a8ae252ec94509ee04d66163d1c7 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Wed, 8 May 2024 13:15:48 -0500 Subject: [PATCH 20/27] Add build instructions to the README file --- Readme.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Readme.md b/Readme.md index 03e8afb..66a6385 100644 --- a/Readme.md +++ b/Readme.md @@ -3,3 +3,52 @@ This repository is to provide libraries needed to compile FreeCAD under Windows. The LibPack is tested to work with [Microsoft Visual C++](https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B) (a.k.a. MSVC or VC). It should be possible to use other compilers like MinGW, however this is not tested. For information how to use the LibPack to compile, see this Wiki page: https://wiki.freecadweb.org/Compile_on_Windows + +## Building the LibPack ## + +To build the LibPack locally, you will need the following: + * Network access + * A working compiler toolchain for your system, accessible by cMake + * CMake + * git + * 7z (see https://www.7-zip.org) + * Python >= 3.8 (**not** used inside the LibPack itself, just used to run the creation script) + * The "requests" Python package (e.g. 'pip install requests') + * The "diff-match-patch" Python package (e.g. 'pip install diff-match-patch') + * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF + * NOTE: The two Web requirements will be dropped from the LibPack before it is released + * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) + +With those pieces in place, the next step is to configure the contents of the LibPack by editing `config.json`. This file +lists the source for each LibPack component. Depending on the component, there are three different ways it might be included: +1) Source code checked out from a git repository and built using the local compiler toolchain +1) A pip package installed to the LibPack directory using the LibPack's Python interpreter + * Note that `pip` itself is installed using the `ensure_pip` Python module +1) Files copied from a local source + +The JSON file just lists out the sources and versions: beyond specifying which method is used for the installation by setting +either "git-repo" and "git_ref", "pip-install", or "install-directory", the actual details of how things are built when source +code is provided are set in the `compile_all.py` script. In that file, the class `Compiler` contains methods following the +naming convention `build_XXX` where `XXX` is the "name" provided in the JSON configuration file. If you need to add a compiled +or copied package, you must both specify it in the config.json file and provide a matching `build_XXX` method. For pip +installation, only the config.json file needs to be edited to include the new dependency. + +To change the way a package is compiled, you edit its entry in `compile_all.py`. See the contents of that file for various +examples. + +**NOTE**: In this prerelease LibPack 3.0 builder, you will need to apply the contents of https://github.com/FreeCAD/FreeCAD/pull/10337 to your FreeCAD build. + +## Running the build script ## + +``` +python.exe create_libpack [arguments] +``` +Arguments: +* `-m`, `--mode` -- 'release' or 'debug' (Default: 'release') +* `-c`, `--config` -- Path to a JSON configuration file for this utility (Default: './config.json') +* `-w`, `--working` -- Directory to put all the clones and downloads in (Default: './working') +* `-e`, `--no-skip-existing-clone` -- If a given clone (or download) directory exists, delete it and download it again +* `-b`, `--no-skip-existing-build` -- If a given build already exists, run the build process again anyway +* `-s`, `--silent` -- I kow what I'm doing, don't ask me any questions +* `--7zip` -- Path to 7-zip executable +* `--bison` -- Path to Bison executable From 956c78a3c75e4f1f1db94c41981a8d455f6fdd0e Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Mon, 3 Jun 2024 21:52:30 -0500 Subject: [PATCH 21/27] Update versions --- config.json | 40 ++++++++++++++++++++-------------------- create_libpack.py | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/config.json b/config.json index 8c45bc1..b7e5def 100644 --- a/config.json +++ b/config.json @@ -5,7 +5,7 @@ { "name":"python", "git-repo":"https://github.com/python/cpython.git", - "git-ref":"v3.11.5" + "git-ref":"v3.12.3" }, { "name":"pip", @@ -13,23 +13,23 @@ }, { "name":"setuptools", - "pip-install":"setuptools==68.2.0" + "pip-install":"setuptools==70.0.0" }, { "name":"wheel", - "pip-install":"wheel==0.41.2" + "pip-install":"wheel==0.43.0" }, { "name":"numpy", - "pip-install":"numpy==1.25.2" + "pip-install":"numpy==1.26.4" }, { "name":"scipy", - "pip-install":"scipy==1.11.2" + "pip-install":"scipy==1.13.1" }, { "name":"pillow", - "pip-install":"Pillow==10.0.0" + "pip-install":"Pillow==10.3.0" }, { "name":"pyyaml", @@ -37,19 +37,19 @@ }, { "name":"pycollada", - "pip-install":"pycollada==0.7.2" + "pip-install":"pycollada==0.8" }, { "name":"matplotlib", - "pip-install":"matplotlib==3.7.2" + "pip-install":"matplotlib==3.9.0" }, { "name":"opencv", - "pip-install":"opencv-python==4.8.0.76" + "pip-install":"opencv-python==4.10.0.82" }, { "name":"cmake", - "pip-install":"cmake==3.27.2" + "pip-install":"cmake==3.29.4" }, { "name":"zlib", @@ -58,7 +58,7 @@ }, { "name":"qt", - "install-directory":"C:\\Qt\\6.5.2\\msvc2019_64" + "install-directory":"C:\\Qt\\6.7.1\\msvc2019_64" }, { "name":"bzip2", @@ -73,7 +73,7 @@ { "name":"boost", "git-repo":"https://github.com/boostorg/boost", - "git-ref":"boost-1.83.0" + "git-ref":"boost-1.85.0" }, { "name":"coin", @@ -108,17 +108,17 @@ { "name":"pyside", "git-repo": "http://code.qt.io/pyside/pyside-setup", - "git-ref": "v6.5.2" + "git-ref": "v6.7.1" }, { "name":"vtk", "git-repo":"https://gitlab.kitware.com/vtk/vtk.git", - "git-ref":"v9.2.6" + "git-ref":"v9.3.0" }, { "name":"harfbuzz", "git-repo":"https://github.com/harfbuzz/harfbuzz", - "git-ref":"8.1.1" + "git-ref":"8.5.0" }, { "name":"freetype", @@ -143,12 +143,12 @@ { "name":"opencascade", "git-repo":"https://gitlab.com/blobfish/occt", - "git-ref":"V7_7_1_BF" + "git-ref":"V7_8_0_BF" }, { "name":"netgen", "git-repo":"https://github.com/NGSolve/netgen", - "git-ref":"v6.2.2304", + "git-ref":"v6.2.2403", "patches":["patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch"] }, { @@ -174,17 +174,17 @@ { "name":"icu", "git-repo":"https://github.com/unicode-org/icu", - "git-ref":"release-73-2" + "git-ref":"release-75-1" }, { "name":"xercesc", "git-repo":"https://github.com/apache/xerces-c", - "git-ref":"v3.2.4" + "git-ref":"v3.2.5" }, { "name":"libfmt", "git-repo":"https://github.com/fmtlib/fmt", - "git-ref":"10.1.0" + "git-ref":"10.2.1" }, { "name":"eigen3", diff --git a/create_libpack.py b/create_libpack.py index ac64109..22271ea 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -42,7 +42,7 @@ try: import diff_match_patch except ImportError: - print("Please pip --install diff_match_patch") + print("Please pip --install diff-match-patch") exit(1) import compile_all From 4a3bd3aa7938dad02d8fe127adbeecad8c553238 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Fri, 7 Jun 2024 21:51:11 -0500 Subject: [PATCH 22/27] Update versions and modify scripts accordingly. --- compile_all.py | 113 +++++++++++++----- config.json | 28 +++-- create_libpack.py | 16 +-- patches/medfile-01-const_cast_argv.patch | 25 ++++ patches/vtk-01-add-missing-typename.patch | 5 + .../vtk-02-add_cast_to_specific_type.patch | 7 ++ 6 files changed, 144 insertions(+), 50 deletions(-) create mode 100644 patches/medfile-01-const_cast_argv.patch create mode 100644 patches/vtk-01-add-missing-typename.patch create mode 100644 patches/vtk-02-add_cast_to_specific_type.patch diff --git a/compile_all.py b/compile_all.py index b8d41f9..5b5ef83 100644 --- a/compile_all.py +++ b/compile_all.py @@ -127,6 +127,11 @@ def __init__(self, config, bison_path, skip_existing: bool = False, mode: BuildM self.init_script = None self.mode = mode + # Right now there are two packages where the version number gets coded into the path when: Boost and Coin: + # store those two separately from all the other paths we have to track + self.boost_include_path = None + self.coin_cmake_path = None + def get_cmake_options(self) -> List[str]: """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply to all builds, but none conflict. """ @@ -136,6 +141,8 @@ def get_cmake_options(self) -> List[str]: pcre_lib += to_static() base = [ + '-D CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE', # Never use system packages, always use only the libpack + '-D CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=TRUE', # Same as above? f'-D BISON_EXECUTABLE={self.bison_path}', f'-D BOOST_ROOT={self.install_dir}', f'-D BUILD_DOC=No', @@ -148,12 +155,10 @@ def get_cmake_options(self) -> List[str]: f'-D BUILD_TESTS=No', f'-D BUILD_TESTING=No', f'-D BZIP2_DIR={self.install_dir}/lib/cmake/', - f'-D Boost_INCLUDE_DIR={self.install_dir}/include/boost-1_83/', # TODO Remove hardcoded version f'-D Boost_INCLUDE_DIRS={self.install_dir}/include', f'-D CMAKE_BUILD_TYPE={self.mode}', f'-D CMAKE_INSTALL_PATH={self.install_dir}', f'-D CMAKE_INSTALL_PREFIX={self.install_dir}', - f'-D Coin_DIR={self.install_dir}/lib/cmake/Coin-4.0.1', f'-D HarfBuzz_DIR={self.install_dir}/lib/cmake/', f'-D HDF5_DIR={self.install_dir}/share/cmake/', f'-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib', @@ -161,6 +166,7 @@ def get_cmake_options(self) -> List[str]: f'-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff' + to_exe(), f'-D INSTALL_DIR={self.install_dir}', f'-D PCRE2_LIBRARY={pcre_lib}', + '-D PIVY_USE_QT6=Yes', f'-D Python_ROOT_DIR={self.install_dir}/bin', f'-D Python_DIR={self.install_dir}/bin', '-D Python_FIND_REGISTRY=NEVER', @@ -172,10 +178,17 @@ def get_cmake_options(self) -> List[str]: f'-D ZLIB_INCLUDE_DIR={self.install_dir}/include', f'-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib' + to_static(), f'-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd' + to_static(), + '-D CMAKE_DISABLE_FIND_PACKAGE_SoQt=True', # Absolutely never find SoQt (it's deprecated and we don't want it!) ] + if self.boost_include_path: + base.append(f'-D Boost_INCLUDE_DIR={self.boost_include_path}') + if self.coin_cmake_path: + base.append(f'-D Coin_DIR={self.coin_cmake_path}') if sys.platform.startswith('win32'): inc_path = self.install_dir.replace('\\', '/') - cxx_flags = f'/I{inc_path}/include /EHsc /DWIN32' + cxx_flags = f'/I{inc_path}/include /EHsc /DWIN32 /Zc:__cplusplus /std:c++17 /permissive-' + # NOTE: /permissive- is required with Qt6 but could be disabled for anything that doesn't link against Qt + # The same is true for /Zc:__cplusplus /std:c++17 else: cxx_flags = f'-I{self.install_dir}/include' base.append(f'-D CMAKE_CXX_FLAGS={cxx_flags}') @@ -249,7 +262,7 @@ def build_python(self, _=None): os.makedirs(libs_dir, exist_ok=True) os.makedirs(bin_dir, exist_ok=True) os.makedirs(tools_dir, exist_ok=True) - tools_subs = ["i18n", "scripts", "demo"] + tools_subs = ["i18n", "scripts"] for sub in tools_subs: os.makedirs(os.path.join(tools_dir, sub), exist_ok=True) @@ -343,7 +356,7 @@ def get_python_version(self, exe: str = None) -> str: def build_pip(self, _=None): path_to_python = self.python_exe() try: - subprocess.run([path_to_python, "-m", "ensurepip"], capture_output=True, check=True) + subprocess.run([path_to_python, "-m", "ensurepip", "--upgrade"], capture_output=True, check=True) except subprocess.CalledProcessError as e: print("ERROR: Failed to run LibPack's Python executable") print(e.stdout.decode("utf-8")) @@ -366,7 +379,8 @@ def build_qt(self, options: dict): def build_boost(self, _=None): """ Builds boost shared libraries and installs libraries and headers """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "include", "boost-1_83")): + self._configure_boost_version() + if self.boost_include_path is not None: print(" Not rebuilding boost, it is already in the LibPack") return # Boost uses a custom build system and needs a config file to find our Python @@ -377,7 +391,7 @@ def build_boost(self, _=None): inc_dir = os.path.join(self.install_dir, "bin", "include").replace("\\", "\\\\") lib_dir = os.path.join(self.install_dir, "bin", "libs").replace("\\", "\\\\") python_version = self.get_python_version() - full_version = python_version + "d" if self.mode == BuildMode.DEBUG else "" + full_version = python_version + ("d" if self.mode == BuildMode.DEBUG else "") print(f" (boost-python is being built against Python {full_version})") user_config.write(f'using python : {python_version} ') user_config.write(f': "{exe}" ') @@ -388,21 +402,22 @@ def build_boost(self, _=None): user_config.write(";\n") try: # When debugging on the command line, add --debug-configuration to get more verbose output - subprocess.run([self.init_script, "&", "bootstrap.bat"], capture_output=True, check=True) + install_dir = self.install_dir + subprocess.run([self.init_script, "&", "bootstrap.bat", f"--prefix={install_dir}"], capture_output=True, check=True) subprocess.run([self.init_script, "&", "b2", f"install", "address-model=64", "link=static,shared", str(self.mode).lower(), - "python-debugging=" + "on" if self.mode == BuildMode.DEBUG else "off", - f"--prefix=${self.install_dir}", + "python-debugging=" + ("on" if self.mode == BuildMode.DEBUG else "off"), + f"--prefix={install_dir}", "--layout=versioned", + "--without-mpi", + "--without-graph_parallel", "--build-type=complete", - f"stage"], + "--debug-configuration"], check=True, capture_output=True) - # NOTE: I get an error when running this here, but when I run what I believe is the same command from a - # command line, it works fine. except subprocess.CalledProcessError as e: # Boost is too verbose in its output to be of much use un-processed. Dump it all to a file, and # then print only the lines with the word "error" on them to stdout @@ -415,6 +430,16 @@ def build_boost(self, _=None): # Lots of these lines are just files with the word 'error' in them, maybe there is a better filter? print(line) exit(e.returncode) + self._configure_boost_version() + + def _configure_boost_version(self): + """ Once Boost has been installed, figure out what version it was and set up the correct include path """ + start_crawl_at = os.path.join(self.install_dir, "include") + contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + for item in contents: + if item.startswith("boost"): + self.boost_include_path = os.path.join(start_crawl_at, item) + break def _cmake_create_build_dir(self): build_dir = "build-" + str(self.mode).lower() @@ -439,7 +464,7 @@ def _cmake_configure(self, extra_args: List[str] = None): options = self.get_cmake_options() if extra_args: options.extend(extra_args) - options.append("..") + options.append("..") # Because the source code is located one directory up from our build location self._run_cmake(options) def _cmake_build(self, parallel: bool = True): @@ -477,11 +502,22 @@ def _build_with_pip(self, options: dict): def build_coin(self, _=None): """ Builds and installs Coin using standard CMake settings """ if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "share", "Coin")): + self._configure_coin_cmake_path() + if self.coin_cmake_path is not None: print(" Not rebuilding Coin, it is already in the LibPack") return - extra_args = ["-DCOIN_BUILD_TESTS=Off"] + extra_args = ["-D COIN_BUILD_TESTS=Off"] self._build_standard_cmake(extra_args) + self._configure_coin_cmake_path() + + def _configure_coin_cmake_path(self): + """ Coin installs its cMake file into a directory named with the full version, so figure out what that is """ + start_crawl_at = os.path.join(self.install_dir, "lib", "cmake") + contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + for item in contents: + if item.startswith("Coin"): + self.coin_cmake_path = os.path.join(start_crawl_at, item) + break def build_quarter(self, _=None): """ Builds and installs Quarter using standard CMake settings """ @@ -541,10 +577,9 @@ def build_pivy(self, _=None): return self._build_standard_cmake() if self.mode == BuildMode.DEBUG: - base = os.path.join(self.install_dir,"bin","Lib","site-packages","pivy") + base = os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy") os.rename(os.path.join(base, "_coin.pyd"), os.path.join(base, "_coin_d.pyd")) - def build_libclang(self, _=None): """ libclang is provided as a platform-specific download by Qt. """ if self.skip_existing: @@ -715,19 +750,38 @@ def build_rapidjson(self, _): shutil.rmtree(os.path.join(self.install_dir, "include", "rapidjson")) shutil.copytree("include", os.path.join(self.install_dir, "include"), dirs_exist_ok=True) + def _get_vtk_include_path(self) ->str: + """ + OpenCASCADE needs a manually-set include path for VTK (the find_package script provided by VTK does not provide + the include file path, and OpenCASCADE has not been updated to handle this, as of June 2024). + """ + start_crawl_at = os.path.join(self.install_dir, "include") + contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + for item in contents: + if item.startswith("vtk-"): + return os.path.join(start_crawl_at, item) + raise RuntimeError("Could not find VTK include directory for OpenCASCADE") + def build_opencascade(self, _=None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): print(" Not rebuilding OpenCASCADE, it is already in the LibPack") return - extra_args = [f"-D 3RDPARTY_DIR={self.install_dir}", - f"-D 3RDPARTY_VTK_INCLUDE_DIR={self.install_dir}/include/vtk-9.2", # TODO: Remove hardcoded 9.2 - f"-D USE_VTK=On", - f"-D USE_RAPIDJSON=On", - f"-D BUILD_CPP_STANDARD=C++17", - f"-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", - f"-D INSTALL_DIR_BIN=bin", - f"-D INSTALL_DIR_LIB=lib"] + extra_args = [f"-D CMAKE_MODULE_PATH={self.install_dir}/lib/cmake;{self.install_dir}/share/cmake;{self.install_dir}" + f"-D TCL_DIR={self.install_dir}/include", + f"-D TK_DIR={self.install_dir}/include", + f"-D FREETYPE_DIR={self.install_dir}/lib/cmake", + f"-D VTK_DIR={self.install_dir}/lib/cmake", + f"-D 3RDPARTY_VTK_INCLUDE_DIRS={self._get_vtk_include_path()}", + f"-D EIGEN_DIR={self.install_dir}/share/eigen3/cmake", + "-D USE_VTK=On", + "-D USE_FREETYPE=On" + "-D USE_RAPIDJSON=On", + "-D USE_EIGEN=On" + "-D BUILD_CPP_STANDARD=C++17", + "-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", + "-D INSTALL_DIR_BIN=bin", + "-D INSTALL_DIR_LIB=lib"] if self.mode == BuildMode.DEBUG: extra_args.append("-D BUILD_SHARED_LIBRARY_NAME_POSTFIX=d") cwd = os.getcwd() @@ -744,7 +798,7 @@ def build_opencascade(self, _=None): # TODO - something is getting messed up in the CMake config output (note the quotes around 26812): for now just # drop the line entirely - # set (OpenCASCADE_CXX_FLAGS "/IG:/FreeCAD/FreeCAD-LibPack-chennes/working/LibPack-0.22-v3.0.0-Debug/include /EHa /fp:precise /fp:precise /wd"26812" /MP /W4") + # set (OpenCASCADE_CXX_FLAGS "[...] /wd"26812" /MP /W4") with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "r", encoding="utf-8") as f: occt_cmake_contents = f.readlines() with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "w", encoding="utf-8") as f: @@ -766,7 +820,8 @@ def build_netgen(self, _: None): f"-D TK_DIR={self.install_dir}", "-D USE_OCC=On", f"-D OpenCASCADE_ROOT={self.install_dir}", - f"-D USE_PYTHON=OFF"] + f"-D USE_PYTHON=OFF", + f"-D CMAKE_CXX_FLAGS=-D_USE_MATH_DEFINES"] # To get M_PI on MSVC self._build_standard_cmake(extra_args=extra_args) def build_hdf5(self, _: None): @@ -843,7 +898,7 @@ def build_xercesc(self, _: None): if os.path.exists(os.path.join(self.install_dir, "include", "xercesc")): print(" Not rebuilding xerces-c, it is already in the LibPack") return - extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include/unicode", + extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include", f"-D ICU_ROOT={self.install_dir}", f"-D ICU_UC_DIR={self.install_dir}"] self._build_standard_cmake(extra_args) diff --git a/config.json b/config.json index b7e5def..b9b9641 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "FreeCAD-version":"0.22", + "FreeCAD-version":"1.0.0", "LibPack-version":"3.0.0", "content": [ { @@ -49,7 +49,7 @@ }, { "name":"cmake", - "pip-install":"cmake==3.29.4" + "pip-install":"cmake==3.29.3" }, { "name":"zlib", @@ -113,7 +113,8 @@ { "name":"vtk", "git-repo":"https://gitlab.kitware.com/vtk/vtk.git", - "git-ref":"v9.3.0" + "git-ref":"v9.3.0", + "patches": ["patches/vtk-01-add-missing-typename.patch", "patches/vtk-02-add_cast_to_specific_type.patch"] }, { "name":"harfbuzz", @@ -140,6 +141,11 @@ "git-repo":"https://github.com/Tencent/rapidjson", "git-ref":"v1.1.0" }, + { + "name":"eigen3", + "git-repo":"https://gitlab.com/libeigen/eigen", + "git-ref":"3.4.0" + }, { "name":"opencascade", "git-repo":"https://gitlab.com/blobfish/occt", @@ -148,8 +154,7 @@ { "name":"netgen", "git-repo":"https://github.com/NGSolve/netgen", - "git-ref":"v6.2.2403", - "patches":["patches/netgen-01-compiler_bug_workaround_msvc14_std_atomic.patch"] + "git-ref":"v6.2.2403" }, { "name":"hdf5", @@ -159,12 +164,13 @@ { "name":"medfile", "git-repo":"https://github.com/chennes/med", - "git-ref":"v4.1.1" + "git-ref":"v4.1.1", + "patches": ["patches/medfile-01-const_cast_argv.patch"] }, { "name":"gmsh", "git-repo":"https://gitlab.onelab.info/gmsh/gmsh", - "git-ref":"gmsh_4_11_1" + "git-ref":"gmsh_4_13_1" }, { "name":"pycxx", @@ -174,7 +180,8 @@ { "name":"icu", "git-repo":"https://github.com/unicode-org/icu", - "git-ref":"release-75-1" + "git-ref":"release-74-2", + "note":"Cannot yet use the 75.x series, compilation of Xerces 3.2.5 fails against them" }, { "name":"xercesc", @@ -186,11 +193,6 @@ "git-repo":"https://github.com/fmtlib/fmt", "git-ref":"10.2.1" }, - { - "name":"eigen3", - "git-repo":"https://gitlab.com/libeigen/eigen", - "git-ref":"3.4.0" - }, { "name": "yamlcpp", "git-repo": "https://github.com/jbeder/yaml-cpp", diff --git a/create_libpack.py b/create_libpack.py index 22271ea..28c2f5a 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -16,14 +16,14 @@ # * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) # Note about Python: Python includes the following dependencies when built on Windows (as of v3.11.5) -# bzip2-1.0.8 -# sqlite-3.42.0.0 -# xz-5.2.5 -# zlib-1.2.13 -# libffi-3.4.4 -# openssl-bin-3.0.9 -# tcltk-8.6.12.1 -# At present these are not re-used +# bzip2 +# sqlite +# xz +# zlib +# libffi +# openssl-bin +# tcltk +# At present these are not re-used to create the rest of the LibPack -- if needed, they are rebuilt from source import argparse import json diff --git a/patches/medfile-01-const_cast_argv.patch b/patches/medfile-01-const_cast_argv.patch new file mode 100644 index 0000000..fd7c32f --- /dev/null +++ b/patches/medfile-01-const_cast_argv.patch @@ -0,0 +1,25 @@ +@@@ tools/medimport/medimportcxx.cxx @@@ +@@ -836,26 +836,8 @@ + %7B%0A%0A +- char * fileOut;%0A + in +@@ -1000,57 +1000,174 @@ + %7D%0A +-if (argc == 2 ) fileOut=%22%22; else ++char fileOut%5B4096%5D;%0A if (argc == 2 ) %7B%0A strncpy(fileOut, %22%22, 1);%0A %7D else %7B%0A strncpy( + fileOut +-= ++, + argv%5B2%5D +-; ++, strlen(argv%5B2%5D)+1); // Include the terminating null%0A %7D + %0A%0A +@@ -1195,16 +1195,36 @@ + port ++(const_cast%3Cchar *%3E + (argv%5B1%5D + , fi +@@ -1219,16 +1219,17 @@ + (argv%5B1%5D ++) + , fileOu diff --git a/patches/vtk-01-add-missing-typename.patch b/patches/vtk-01-add-missing-typename.patch new file mode 100644 index 0000000..e7141ec --- /dev/null +++ b/patches/vtk-01-add-missing-typename.patch @@ -0,0 +1,5 @@ +@@@ Common/Math/vtkFFT.txx @@@ +@@ -11823,16 +11823,25 @@ + pleRef = ++ typename + decltyp diff --git a/patches/vtk-02-add_cast_to_specific_type.patch b/patches/vtk-02-add_cast_to_specific_type.patch new file mode 100644 index 0000000..bda6bd8 --- /dev/null +++ b/patches/vtk-02-add_cast_to_specific_type.patch @@ -0,0 +1,7 @@ +@@@ Charts/Core/vtkPlotBag.cxx @@@ +@@ -8951,19 +8951,33 @@ + ing() : ++vtkStdString( + %22?%22 ++) + ;%0A From 4c4aa77d0f44a9493101436661b179334a989142 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Thu, 20 Jun 2024 07:48:37 -0500 Subject: [PATCH 23/27] Cleanup and tweaks --- compile_all.py | 8 ++++++ config.json | 9 +++++++ path_cleaner.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 path_cleaner.py diff --git a/compile_all.py b/compile_all.py index 5b5ef83..ba59b93 100644 --- a/compile_all.py +++ b/compile_all.py @@ -924,3 +924,11 @@ def build_yamlcpp(self, _: None): return extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"] self._build_standard_cmake(extra_args) + + def build_opencamlib(self, _:None): + if self.skip_existing: + if os.path.exists(os.path.join(self.install_dir, "include", "opencamlib")): + print(" Not rebuilding opencamlib, it is already in the LibPack") + return + extra_args = ["-D CXX_LIB=ON"] + self._build_standard_cmake(extra_args) diff --git a/config.json b/config.json index b9b9641..26c39ed 100644 --- a/config.json +++ b/config.json @@ -51,6 +51,10 @@ "name":"cmake", "pip-install":"cmake==3.29.3" }, + { + "name":"ifcopenshell", + "pip-install":"ifcopenshell==0.7.0.240521" + }, { "name":"zlib", "git-repo":"https://github.com/madler/zlib", @@ -197,6 +201,11 @@ "name": "yamlcpp", "git-repo": "https://github.com/jbeder/yaml-cpp", "git-ref":"0.8.0" + }, + { + "name": "opencamlib", + "git-repo": "https://github.com/aewallin/opencamlib", + "git-ref": "2023.01.11" } ] } diff --git a/path_cleaner.py b/path_cleaner.py new file mode 100644 index 0000000..bdcc16e --- /dev/null +++ b/path_cleaner.py @@ -0,0 +1,66 @@ +# What I really want to do is clean for release. So replace explicit paths with references to CMAKE_CURRENT_SOURCE_DIR +# in cMake files, and also delete some extra files that are spewed out by various installers. The various licenses +# should probably be consolidated. + +import os + +paths_to_delete = [ + "custom_vc14_64.bat", + "custom.bat", + "USING_HDF5_CMake.txt", + "USING_HDF5_VS.txt", + "env.bat", + "draw.bat", + "RELEASE.txt", + ] + + +def delete_extraneous_files(base_path: str) -> None: + """Delete each of the files listed above from the path specified in base_path. Failure to delete a file does not + constitute a fatal error.""" + if not os.path.exists(base_path): + raise RuntimeError(f"{base_path} does not exist") + if not os.path.isdir(base_path): + raise RuntimeError(f"{base_path} is not a directory") + for file in paths_to_delete: + try: + os.unlink(os.path.join(base_path, file)) + except OSError as e: + print(e) + + +def remove_local_path_from_cmake_files(base_path: str) -> None: + """In many cases, the local compilation paths get stored into the cMake files. They should not ever be used, but + a) OpenCASCADE codes in the local path to FreeType, which then fails when the LibPack is distributed, and b) for + good measure cMake files shouldn't refer to non-existent paths on a foreign system. So this method looks for + cmake config files and cleans the ones it finds.""" + for root, dirs, files in os.walk(base_path): + for file in files: + if file.lower().endswith(".cmake"): + remove_local_path_from_cmake_file(base_path, os.path.join(root, file)) + + +def remove_local_path_from_cmake_file(base_path: str, file_to_clean: str) -> None: + """ Modify a cMake file to remove base_path and replace it with ${CMAKE_CURRENT_SOURCE_DIR} -- WARNING: effectively + edits the file in-place, no backup is made.""" + depth_string = create_depth_string(base_path, file_to_clean) + with open(file_to_clean, "r", encoding="UTF-8") as f: + contents = f.read() + contents.replace(base_path, "${CMAKE_CURRENT_SOURCE_DIR}/" + depth_string) + with open(file_to_clean, "w", encoding="utf-8") as f: + f.write(contents) + + +def create_depth_string(base_path: str, file_to_clean: str) -> str: + """Given a base path and a file, determine how many "../" must be appended to the file's containing directory + to result in a path that resolves to base_path. Returns a string containing just some number of occurrences of + "../" e.g. "../../../" to move up three levels from file_to_clean's containing folder.""" + if not file_to_clean.startswith(base_path): + raise RuntimeError(f"{file_to_clean} does not appear to be in {base_path}") + + containing_directory = os.path.dirname(file_to_clean) + directories_to_file = len(containing_directory.split(os.path.sep)) + directories_in_base = len(base_path.split(os.path.sep)) + num_steps_up = directories_to_file - directories_in_base + return "../" * num_steps_up # For use in cMake, so always a forward slash here + From 4de4cf547c1af7c7a4a8f4163d63278945314a2b Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Thu, 20 Jun 2024 08:16:05 -0500 Subject: [PATCH 24/27] Add pre-commit config --- .pre-commit-config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..645b37d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: mixed-line-ending +- repo: https://github.com/psf/black + rev: 24.3.0 + hooks: + - id: black + args: ['--line-length', '100'] From 564c1389f8a9ba8fce9bfb299987f8230de6786f Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Thu, 20 Jun 2024 08:22:57 -0500 Subject: [PATCH 25/27] Reformat per pre-commit settings --- .pre-commit-config.yaml | 2 + Readme.md | 6 +- compile_all.py | 439 ++++++++++++++++++++++++---------------- create_libpack.py | 20 +- generate_patch.py | 4 +- path_cleaner.py | 7 +- test_compile_all.py | 16 +- test_create_libpack.py | 16 +- test_generate_patch.py | 9 +- 9 files changed, 302 insertions(+), 217 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 645b37d..7007500 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,8 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +exclude: 'patches/.*' + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 diff --git a/Readme.md b/Readme.md index 66a6385..4a5193d 100644 --- a/Readme.md +++ b/Readme.md @@ -16,14 +16,14 @@ To build the LibPack locally, you will need the following: * The "requests" Python package (e.g. 'pip install requests') * The "diff-match-patch" Python package (e.g. 'pip install diff-match-patch') * Qt - the base installation plus Qt Image Formats, Qt Webengine, Qt Webview, and Qt PDF - * NOTE: The two Web requirements will be dropped from the LibPack before it is released + * NOTE: The two Web requirements will be dropped from the LibPack before it is released * GNU Bison (for Windows see https://github.com/lexxmark/winflexbison/) -With those pieces in place, the next step is to configure the contents of the LibPack by editing `config.json`. This file +With those pieces in place, the next step is to configure the contents of the LibPack by editing `config.json`. This file lists the source for each LibPack component. Depending on the component, there are three different ways it might be included: 1) Source code checked out from a git repository and built using the local compiler toolchain 1) A pip package installed to the LibPack directory using the LibPack's Python interpreter - * Note that `pip` itself is installed using the `ensure_pip` Python module + * Note that `pip` itself is installed using the `ensure_pip` Python module 1) Files copied from a local source The JSON file just lists out the sources and versions: beyond specifying which method is used for the installation by setting diff --git a/compile_all.py b/compile_all.py index ba59b93..77d8088 100644 --- a/compile_all.py +++ b/compile_all.py @@ -71,7 +71,7 @@ def split_patch_data(patch_data: str) -> List[Dict[str, str]]: def apply_patch(patch_file_path: str) -> None: - """ Apply a patch that was generated by the generate_patch.py script """ + """Apply a patch that was generated by the generate_patch.py script""" # Path is relative to *this* file, not our working directory absolute_path = os.path.join(pathlib.Path(__file__).parent.absolute(), patch_file_path) with open(absolute_path, "r", encoding="utf-8") as f: @@ -82,7 +82,7 @@ def apply_patch(patch_file_path: str) -> None: def patch_files(patches: List[str]) -> None: - """ Given a list of patches, apply them sequentially in the current working directory. The patches themselves are + """Given a list of patches, apply them sequentially in the current working directory. The patches themselves are expected to be given as paths relative to **this** Python script file""" for patch in patches: start = len("patches/") @@ -92,33 +92,33 @@ def patch_files(patches: List[str]) -> None: def libpack_dir(config: dict, mode: BuildMode): lp_dir = "LibPack-{}-v{}-{}".format( - config["FreeCAD-version"], - config["LibPack-version"], - str(mode) + config["FreeCAD-version"], config["LibPack-version"], str(mode) ) return os.path.join(os.path.dirname(__file__), "working", lp_dir) def to_exe(base: str = ""): - """ Append .exe to Windows executables, but not to macOS or Linux. If given no argument, just returns the extension - for the current OS, suitable for appending to an executable's name. """ - return base + '.exe' if sys.platform.startswith('win32') else '' + """Append .exe to Windows executables, but not to macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to an executable's name.""" + return base + ".exe" if sys.platform.startswith("win32") else "" def to_static(base: str = ""): - """ Append .lib to Windows libraries, or .a macOS or Linux. If given no argument, just returns the extension - for the current OS, suitable for appending to a static library's name. """ - return base + '.lib' if sys.platform.startswith('win32') else '.a' + """Append .lib to Windows libraries, or .a macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to a static library's name.""" + return base + ".lib" if sys.platform.startswith("win32") else ".a" def to_dynamic(base: str = ""): - """ Append .dll to Windows libraries, or .so to macOS or Linux. If given no argument, just returns the extension - for the current OS, suitable for appending to a dynamic library's name. """ - return base + '.dll' if sys.platform.startswith('win32') else '.so' + """Append .dll to Windows libraries, or .so to macOS or Linux. If given no argument, just returns the extension + for the current OS, suitable for appending to a dynamic library's name.""" + return base + ".dll" if sys.platform.startswith("win32") else ".so" class Compiler: - def __init__(self, config, bison_path, skip_existing: bool = False, mode: BuildMode = BuildMode.RELEASE): + def __init__( + self, config, bison_path, skip_existing: bool = False, mode: BuildMode = BuildMode.RELEASE + ): self.config = config self.bison_path = bison_path self.base_dir = os.getcwd() @@ -133,65 +133,67 @@ def __init__(self, config, bison_path, skip_existing: bool = False, mode: BuildM self.coin_cmake_path = None def get_cmake_options(self) -> List[str]: - """ Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply - to all builds, but none conflict. """ + """Get a comprehensive list of cMake options that can be used in any cMake build. Not all options apply + to all builds, but none conflict.""" pcre_lib = self.install_dir + "/lib/pcre2-8" if self.mode == BuildMode.DEBUG: pcre_lib += "d" pcre_lib += to_static() base = [ - '-D CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE', # Never use system packages, always use only the libpack - '-D CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=TRUE', # Same as above? - f'-D BISON_EXECUTABLE={self.bison_path}', - f'-D BOOST_ROOT={self.install_dir}', - f'-D BUILD_DOC=No', - f'-D BUILD_DOCS=No', - f'-D BUILD_EXAMPLES=No', - f'-D BUILD_SHARED=Yes', - f'-D BUILD_SHARED_LIB=Yes', - f'-D BUILD_SHARED_LIBS=Yes', - f'-D BUILD_TEST=No', - f'-D BUILD_TESTS=No', - f'-D BUILD_TESTING=No', - f'-D BZIP2_DIR={self.install_dir}/lib/cmake/', - f'-D Boost_INCLUDE_DIRS={self.install_dir}/include', - f'-D CMAKE_BUILD_TYPE={self.mode}', - f'-D CMAKE_INSTALL_PATH={self.install_dir}', - f'-D CMAKE_INSTALL_PREFIX={self.install_dir}', - f'-D HarfBuzz_DIR={self.install_dir}/lib/cmake/', - f'-D HDF5_DIR={self.install_dir}/share/cmake/', - f'-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib', - f'-D HDF5_LIBRARY_RELEASE={self.install_dir}/lib/hdf5.lib', - f'-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff' + to_exe(), - f'-D INSTALL_DIR={self.install_dir}', - f'-D PCRE2_LIBRARY={pcre_lib}', - '-D PIVY_USE_QT6=Yes', - f'-D Python_ROOT_DIR={self.install_dir}/bin', - f'-D Python_DIR={self.install_dir}/bin', - '-D Python_FIND_REGISTRY=NEVER', - f'-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6', - f'-D SWIG_EXECUTABLE={self.install_dir}/bin/swig' + to_exe(), - f'-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO', # Workaround for bug in Visual Studio MSVC 143 - f'-D VTK_MODULE_ENABLE_VTK_ioss=NO', # Workaround for bug in Visual Studio MSVC 143 - f'-D ZLIB_DIR={self.install_dir}/lib/cmake/', - f'-D ZLIB_INCLUDE_DIR={self.install_dir}/include', - f'-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib' + to_static(), - f'-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd' + to_static(), - '-D CMAKE_DISABLE_FIND_PACKAGE_SoQt=True', # Absolutely never find SoQt (it's deprecated and we don't want it!) + "-D CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE", # Never use system packages, always use only the libpack + "-D CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=TRUE", # Same as above? + f"-D BISON_EXECUTABLE={self.bison_path}", + f"-D BOOST_ROOT={self.install_dir}", + f"-D BUILD_DOC=No", + f"-D BUILD_DOCS=No", + f"-D BUILD_EXAMPLES=No", + f"-D BUILD_SHARED=Yes", + f"-D BUILD_SHARED_LIB=Yes", + f"-D BUILD_SHARED_LIBS=Yes", + f"-D BUILD_TEST=No", + f"-D BUILD_TESTS=No", + f"-D BUILD_TESTING=No", + f"-D BZIP2_DIR={self.install_dir}/lib/cmake/", + f"-D Boost_INCLUDE_DIRS={self.install_dir}/include", + f"-D CMAKE_BUILD_TYPE={self.mode}", + f"-D CMAKE_INSTALL_PATH={self.install_dir}", + f"-D CMAKE_INSTALL_PREFIX={self.install_dir}", + f"-D HarfBuzz_DIR={self.install_dir}/lib/cmake/", + f"-D HDF5_DIR={self.install_dir}/share/cmake/", + f"-D HDF5_LIBRARY_DEBUG={self.install_dir}/lib/hdf5d.lib", + f"-D HDF5_LIBRARY_RELEASE={self.install_dir}/lib/hdf5.lib", + f"-D HDF5_DIFF_EXECUTABLE={self.install_dir}/bin/hdf5diff" + to_exe(), + f"-D INSTALL_DIR={self.install_dir}", + f"-D PCRE2_LIBRARY={pcre_lib}", + "-D PIVY_USE_QT6=Yes", + f"-D Python_ROOT_DIR={self.install_dir}/bin", + f"-D Python_DIR={self.install_dir}/bin", + "-D Python_FIND_REGISTRY=NEVER", + f"-D Qt6_DIR={self.install_dir}/lib/cmake/Qt6", + f"-D SWIG_EXECUTABLE={self.install_dir}/bin/swig" + to_exe(), + f"-D VTK_MODULE_ENABLE_VTK_IOIOSS=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-D VTK_MODULE_ENABLE_VTK_ioss=NO", # Workaround for bug in Visual Studio MSVC 143 + f"-D ZLIB_DIR={self.install_dir}/lib/cmake/", + f"-D ZLIB_INCLUDE_DIR={self.install_dir}/include", + f"-D ZLIB_LIBRARY_RELEASE={self.install_dir}/lib/zlib" + to_static(), + f"-D ZLIB_LIBRARY_DEBUG={self.install_dir}/lib/zlibd" + to_static(), + "-D CMAKE_DISABLE_FIND_PACKAGE_SoQt=True", # Absolutely never find SoQt (it's deprecated and we don't want it!) ] if self.boost_include_path: - base.append(f'-D Boost_INCLUDE_DIR={self.boost_include_path}') + base.append(f"-D Boost_INCLUDE_DIR={self.boost_include_path}") if self.coin_cmake_path: - base.append(f'-D Coin_DIR={self.coin_cmake_path}') - if sys.platform.startswith('win32'): - inc_path = self.install_dir.replace('\\', '/') - cxx_flags = f'/I{inc_path}/include /EHsc /DWIN32 /Zc:__cplusplus /std:c++17 /permissive-' + base.append(f"-D Coin_DIR={self.coin_cmake_path}") + if sys.platform.startswith("win32"): + inc_path = self.install_dir.replace("\\", "/") + cxx_flags = ( + f"/I{inc_path}/include /EHsc /DWIN32 /Zc:__cplusplus /std:c++17 /permissive-" + ) # NOTE: /permissive- is required with Qt6 but could be disabled for anything that doesn't link against Qt # The same is true for /Zc:__cplusplus /std:c++17 else: - cxx_flags = f'-I{self.install_dir}/include' - base.append(f'-D CMAKE_CXX_FLAGS={cxx_flags}') + cxx_flags = f"-I{self.install_dir}/include" + base.append(f"-D CMAKE_CXX_FLAGS={cxx_flags}") return base def compile_all(self): @@ -207,8 +209,10 @@ def compile_all(self): print(f"Installing {item['name']} with pip") self._build_with_pip(item) else: - print(f"No '{build_function_name}' found in compile_all.py -- " - "did you forget to add one when adding a dependency?") + print( + f"No '{build_function_name}' found in compile_all.py -- " + "did you forget to add one when adding a dependency?" + ) exit(2) os.chdir(self.base_dir) @@ -228,14 +232,17 @@ def build_python(self, _=None): if sys.platform.startswith("win32"): expected_exe_path = self.python_exe() if self.skip_existing and os.path.exists(expected_exe_path): - print("Not rebuilding, instead just using existing Python in the LibPack installation path") + print( + "Not rebuilding, instead just using existing Python in the LibPack installation path" + ) return try: arch = "x64" if platform.machine() == "AMD64" else "ARM64" path = "amd64" if platform.machine() == "AMD64" else "arm64" subprocess.run( [ - self.init_script, "&", + self.init_script, + "&", "PCbuild\\build.bat", "-p", arch, @@ -299,7 +306,9 @@ def build_python(self, _=None): shutil.copytree(f"Tools\\{sub}", os.path.join(tools_dir, sub), dirs_exist_ok=True) # Figure out what version of Python we just built: - major, minor = self.get_python_version(os.path.join("PCBuild", path, "python.exe")).split(".") + major, minor = self.get_python_version( + os.path.join("PCBuild", path, "python.exe") + ).split(".") # Construct the list of files we expect to exist that need to be placed in the toplevel directory, or in # libs: @@ -332,9 +341,7 @@ def build_python(self, _=None): os.unlink(target) os.rename(pyconfig, target) else: - raise NotImplemented( - "Non-Windows compilation of Python is not implemented yet" - ) + raise NotImplemented("Non-Windows compilation of Python is not implemented yet") def get_python_version(self, exe: str = None) -> str: if exe is None: @@ -356,7 +363,9 @@ def get_python_version(self, exe: str = None) -> str: def build_pip(self, _=None): path_to_python = self.python_exe() try: - subprocess.run([path_to_python, "-m", "ensurepip", "--upgrade"], capture_output=True, check=True) + subprocess.run( + [path_to_python, "-m", "ensurepip", "--upgrade"], capture_output=True, check=True + ) except subprocess.CalledProcessError as e: print("ERROR: Failed to run LibPack's Python executable") print(e.stdout.decode("utf-8")) @@ -368,7 +377,9 @@ def build_qt(self, options: dict): qt_dir = options["install-directory"] if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "metatypes")): - print(" Not re-copying, instead just using existing Qt in the LibPack installation path") + print( + " Not re-copying, instead just using existing Qt in the LibPack installation path" + ) return if not os.path.exists(qt_dir): print(f"Error: specified Qt installation path does not exist ({qt_dir})") @@ -377,14 +388,16 @@ def build_qt(self, options: dict): shutil.copytree(qt_dir, self.install_dir, dirs_exist_ok=True) def build_boost(self, _=None): - """ Builds boost shared libraries and installs libraries and headers """ + """Builds boost shared libraries and installs libraries and headers""" if self.skip_existing: self._configure_boost_version() if self.boost_include_path is not None: print(" Not rebuilding boost, it is already in the LibPack") return # Boost uses a custom build system and needs a config file to find our Python - with open(os.path.join("tools", "build", "src", "user-config.jam"), "w", encoding="utf-8") as user_config: + with open( + os.path.join("tools", "build", "src", "user-config.jam"), "w", encoding="utf-8" + ) as user_config: exe = self.python_exe() if sys.platform.startswith("win32"): exe = exe.replace("\\", "\\\\") @@ -393,31 +406,41 @@ def build_boost(self, _=None): python_version = self.get_python_version() full_version = python_version + ("d" if self.mode == BuildMode.DEBUG else "") print(f" (boost-python is being built against Python {full_version})") - user_config.write(f'using python : {python_version} ') + user_config.write(f"using python : {python_version} ") user_config.write(f': "{exe}" ') user_config.write(f': "{inc_dir}" ') user_config.write(f': "{lib_dir}" ') if self.mode == BuildMode.DEBUG: - user_config.write(f': on ') + user_config.write(f": on ") user_config.write(";\n") try: # When debugging on the command line, add --debug-configuration to get more verbose output install_dir = self.install_dir - subprocess.run([self.init_script, "&", "bootstrap.bat", f"--prefix={install_dir}"], capture_output=True, check=True) - subprocess.run([self.init_script, "&", "b2", - f"install", - "address-model=64", - "link=static,shared", - str(self.mode).lower(), - "python-debugging=" + ("on" if self.mode == BuildMode.DEBUG else "off"), - f"--prefix={install_dir}", - "--layout=versioned", - "--without-mpi", - "--without-graph_parallel", - "--build-type=complete", - "--debug-configuration"], - check=True, - capture_output=True) + subprocess.run( + [self.init_script, "&", "bootstrap.bat", f"--prefix={install_dir}"], + capture_output=True, + check=True, + ) + subprocess.run( + [ + self.init_script, + "&", + "b2", + f"install", + "address-model=64", + "link=static,shared", + str(self.mode).lower(), + "python-debugging=" + ("on" if self.mode == BuildMode.DEBUG else "off"), + f"--prefix={install_dir}", + "--layout=versioned", + "--without-mpi", + "--without-graph_parallel", + "--build-type=complete", + "--debug-configuration", + ], + check=True, + capture_output=True, + ) except subprocess.CalledProcessError as e: # Boost is too verbose in its output to be of much use un-processed. Dump it all to a file, and # then print only the lines with the word "error" on them to stdout @@ -433,9 +456,11 @@ def build_boost(self, _=None): self._configure_boost_version() def _configure_boost_version(self): - """ Once Boost has been installed, figure out what version it was and set up the correct include path """ + """Once Boost has been installed, figure out what version it was and set up the correct include path""" start_crawl_at = os.path.join(self.install_dir, "include") - contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + contents = [ + f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f)) + ] for item in contents: if item.startswith("boost"): self.boost_include_path = os.path.join(start_crawl_at, item) @@ -464,7 +489,9 @@ def _cmake_configure(self, extra_args: List[str] = None): options = self.get_cmake_options() if extra_args: options.extend(extra_args) - options.append("..") # Because the source code is located one directory up from our build location + options.append( + ".." + ) # Because the source code is located one directory up from our build location self._run_cmake(options) def _cmake_build(self, parallel: bool = True): @@ -486,8 +513,11 @@ def _build_standard_cmake(self, extra_args: List[str] = None): def _pip_install(self, requirement: str) -> None: path_to_python = self.python_exe() try: - subprocess.run([path_to_python, "-m", "pip", "install", requirement], check=True, - capture_output=True) + subprocess.run( + [path_to_python, "-m", "pip", "install", requirement], + check=True, + capture_output=True, + ) except subprocess.CalledProcessError as e: print(f"ERROR: Failed to pip install {requirement}") print(e.output.decode("utf-8")) @@ -495,12 +525,14 @@ def _pip_install(self, requirement: str) -> None: def _build_with_pip(self, options: dict): if "pip-install" not in options: - print(f"ERROR: No pip-install provided in config of {options['name']}, so version cannot be determined") + print( + f"ERROR: No pip-install provided in config of {options['name']}, so version cannot be determined" + ) exit(1) self._pip_install(options["pip-install"]) def build_coin(self, _=None): - """ Builds and installs Coin using standard CMake settings """ + """Builds and installs Coin using standard CMake settings""" if self.skip_existing: self._configure_coin_cmake_path() if self.coin_cmake_path is not None: @@ -511,16 +543,18 @@ def build_coin(self, _=None): self._configure_coin_cmake_path() def _configure_coin_cmake_path(self): - """ Coin installs its cMake file into a directory named with the full version, so figure out what that is """ + """Coin installs its cMake file into a directory named with the full version, so figure out what that is""" start_crawl_at = os.path.join(self.install_dir, "lib", "cmake") - contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + contents = [ + f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f)) + ] for item in contents: if item.startswith("Coin"): self.coin_cmake_path = os.path.join(start_crawl_at, item) break def build_quarter(self, _=None): - """ Builds and installs Quarter using standard CMake settings """ + """Builds and installs Quarter using standard CMake settings""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "Quarter")): print(" Not rebuilding Quarter, it is already in the LibPack") @@ -535,7 +569,7 @@ def build_zlib(self, _=None): self._build_standard_cmake() def build_bzip2(self, _=None): - """ The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake """ + """The version of BZip2 in widespread use (1.0.8, the most recent official release) do not yet use cMake""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "bzlib.h")): print(" Not rebuilding bzip2, it is already in the LibPack") @@ -546,15 +580,15 @@ def build_bzip2(self, _=None): subprocess.run(args, check=True, capture_output=True) shutil.copyfile("libbz2.lib", os.path.join(self.install_dir, "lib", "libbz2.lib")) shutil.copyfile("bzlib.h", os.path.join(self.install_dir, "include", "bzlib.h")) - shutil.copyfile("bzlib_private.h", os.path.join(self.install_dir, "include", "bzlib_private.h")) + shutil.copyfile( + "bzlib_private.h", os.path.join(self.install_dir, "include", "bzlib_private.h") + ) except subprocess.CalledProcessError as e: print("ERROR: Failed to build bzip2 using nmake") print(e.output.decode("utf-8")) exit(1) else: - raise NotImplemented( - "Non-Windows compilation of bzip2 is not implemented yet" - ) + raise NotImplemented("Non-Windows compilation of bzip2 is not implemented yet") def build_pcre2(self, _=None): if self.skip_existing: @@ -572,7 +606,9 @@ def build_swig(self, _=None): def build_pivy(self, _=None): if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy")): + if os.path.exists( + os.path.join(self.install_dir, "bin", "Lib", "site-packages", "pivy") + ): print(" Not rebuilding pivy, it is already in the LibPack") return self._build_standard_cmake() @@ -581,7 +617,7 @@ def build_pivy(self, _=None): os.rename(os.path.join(base, "_coin.pyd"), os.path.join(base, "_coin_d.pyd")) def build_libclang(self, _=None): - """ libclang is provided as a platform-specific download by Qt. """ + """libclang is provided as a platform-specific download by Qt.""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "clang")): print(" Not copying libclang, it is already in the LibPack") @@ -593,21 +629,37 @@ def build_pyside(self, _=None): # Don't use a pip-install for this, we need the linkable libraries and include files for both PySide and # Shiboken, which won't get installed by pip, and it needs to be built against the right Python exe if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6")): + if os.path.exists( + os.path.join(self.install_dir, "bin", "Lib", "site-packages", "PySide6") + ): print(" Not rebuilding PySide6, it is already in the LibPack") return python = self.python_exe() qtpaths = "--qtpaths=" + os.path.join(self.install_dir, "bin", "qtpaths6") + to_exe() clang = "CLANG_INSTALL_DIR=" + os.path.join(self.install_dir, "lib", "clang") - vulkan = "VULKAN_SDK=None" #"VULKAN_SDK=" + os.path.join(self.install_dir, "Vulkan") + vulkan = "VULKAN_SDK=None" # "VULKAN_SDK=" + os.path.join(self.install_dir, "Vulkan") parallel = "--parallel=16" # numpy = "--enable-numpy-support" if sys.platform.startswith("win32"): ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") - args = [self.init_script, "&", "set", clang, "&", "set", vulkan, "&", python, "setup.py", "install", - qtpaths, ssl, parallel] + args = [ + self.init_script, + "&", + "set", + clang, + "&", + "set", + vulkan, + "&", + python, + "setup.py", + "install", + qtpaths, + ssl, + parallel, + ] if self.mode == BuildMode.DEBUG: - args.append ("--debug") + args.append("--debug") else: ssl = "--openssl=" + os.path.join(self.install_dir, "bin", "DLLs") args = [clang, "&&", python, "setup.py", "install", qtpaths, ssl] @@ -649,8 +701,12 @@ def build_freetype(self, _=None): self._build_standard_cmake() if self.mode == BuildMode.DEBUG: # OCCT *really* wants these libraries named like this: - shutil.copyfile(f"{self.install_dir}/bin/freetyped.dll",f"{self.install_dir}/bin/freetype.dll") - shutil.copyfile(f"{self.install_dir}/lib/freetyped.lib",f"{self.install_dir}/lib/freetype.lib") + shutil.copyfile( + f"{self.install_dir}/bin/freetyped.dll", f"{self.install_dir}/bin/freetype.dll" + ) + shutil.copyfile( + f"{self.install_dir}/lib/freetyped.lib", f"{self.install_dir}/lib/freetype.lib" + ) def force_copy(self, src_components: List[str], dst_components: List[str]): full_src = self.install_dir @@ -667,7 +723,7 @@ def force_copy(self, src_components: List[str], dst_components: List[str]): shutil.copyfile(full_src, full_dst) def build_tcl(self, _=None): - """ tcl does not use cMake """ + """tcl does not use cMake""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "tcl.h")): print(" Not rebuilding tcl, it is already in the LibPack") @@ -679,13 +735,15 @@ def build_tcl(self, _=None): if self.mode == BuildMode.DEBUG: args.append("OPTS=symbols") subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, - "&", - "nmake", - "/f", - "makefile.vc", - "install", - f"INSTALLDIR={self.install_dir}"] + args = [ + self.init_script, + "&", + "nmake", + "/f", + "makefile.vc", + "install", + f"INSTALLDIR={self.install_dir}", + ] if self.mode == BuildMode.DEBUG: args.append("OPTS=symbols") subprocess.run(args, check=True, capture_output=True) @@ -703,12 +761,10 @@ def build_tcl(self, _=None): print(e.stderr.decode("utf-8")) exit(1) else: - raise NotImplemented( - "Non-Windows compilation of tcl is not implemented yet" - ) + raise NotImplemented("Non-Windows compilation of tcl is not implemented yet") def build_tk(self, _=None): - """ tk does not use cMake """ + """tk does not use cMake""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "tk.h")): print(" Not rebuilding tk, it is already in the LibPack") @@ -720,8 +776,15 @@ def build_tk(self, _=None): if self.mode == BuildMode.DEBUG: args.append("OPTS=symbols") subprocess.run(args, check=True, capture_output=True) - args = [self.init_script, "&", "nmake", "/f", "makefile.vc ", "install", - f"INSTALLDIR={self.install_dir}"] + args = [ + self.init_script, + "&", + "nmake", + "/f", + "makefile.vc ", + "install", + f"INSTALLDIR={self.install_dir}", + ] if self.mode == BuildMode.DEBUG: args.append("OPTS=symbols") subprocess.run(args, check=True, capture_output=True) @@ -738,9 +801,7 @@ def build_tk(self, _=None): print(e.output.decode("utf-8")) exit(1) else: - raise NotImplemented( - "Non-Windows compilation of tk is not implemented yet" - ) + raise NotImplemented("Non-Windows compilation of tk is not implemented yet") def build_rapidjson(self, _): if os.path.exists(os.path.join(self.install_dir, "include", "rapidjson")): @@ -750,13 +811,15 @@ def build_rapidjson(self, _): shutil.rmtree(os.path.join(self.install_dir, "include", "rapidjson")) shutil.copytree("include", os.path.join(self.install_dir, "include"), dirs_exist_ok=True) - def _get_vtk_include_path(self) ->str: + def _get_vtk_include_path(self) -> str: """ OpenCASCADE needs a manually-set include path for VTK (the find_package script provided by VTK does not provide the include file path, and OpenCASCADE has not been updated to handle this, as of June 2024). """ start_crawl_at = os.path.join(self.install_dir, "include") - contents = [f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f))] + contents = [ + f for f in os.listdir(start_crawl_at) if os.path.isdir(os.path.join(start_crawl_at, f)) + ] for item in contents: if item.startswith("vtk-"): return os.path.join(start_crawl_at, item) @@ -767,21 +830,21 @@ def build_opencascade(self, _=None): if os.path.exists(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake")): print(" Not rebuilding OpenCASCADE, it is already in the LibPack") return - extra_args = [f"-D CMAKE_MODULE_PATH={self.install_dir}/lib/cmake;{self.install_dir}/share/cmake;{self.install_dir}" - f"-D TCL_DIR={self.install_dir}/include", - f"-D TK_DIR={self.install_dir}/include", - f"-D FREETYPE_DIR={self.install_dir}/lib/cmake", - f"-D VTK_DIR={self.install_dir}/lib/cmake", - f"-D 3RDPARTY_VTK_INCLUDE_DIRS={self._get_vtk_include_path()}", - f"-D EIGEN_DIR={self.install_dir}/share/eigen3/cmake", - "-D USE_VTK=On", - "-D USE_FREETYPE=On" - "-D USE_RAPIDJSON=On", - "-D USE_EIGEN=On" - "-D BUILD_CPP_STANDARD=C++17", - "-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", - "-D INSTALL_DIR_BIN=bin", - "-D INSTALL_DIR_LIB=lib"] + extra_args = [ + f"-D CMAKE_MODULE_PATH={self.install_dir}/lib/cmake;{self.install_dir}/share/cmake;{self.install_dir}" + f"-D TCL_DIR={self.install_dir}/include", + f"-D TK_DIR={self.install_dir}/include", + f"-D FREETYPE_DIR={self.install_dir}/lib/cmake", + f"-D VTK_DIR={self.install_dir}/lib/cmake", + f"-D 3RDPARTY_VTK_INCLUDE_DIRS={self._get_vtk_include_path()}", + f"-D EIGEN_DIR={self.install_dir}/share/eigen3/cmake", + "-D USE_VTK=On", + "-D USE_FREETYPE=On" "-D USE_RAPIDJSON=On", + "-D USE_EIGEN=On" "-D BUILD_CPP_STANDARD=C++17", + "-D BUILD_RELEASE_DISABLE_EXCEPTIONS=OFF", + "-D INSTALL_DIR_BIN=bin", + "-D INSTALL_DIR_LIB=lib", + ] if self.mode == BuildMode.DEBUG: extra_args.append("-D BUILD_SHARED_LIBRARY_NAME_POSTFIX=d") cwd = os.getcwd() @@ -791,7 +854,9 @@ def build_opencascade(self, _=None): if self.mode == BuildMode.DEBUG and sys.platform.startswith("win32"): # On Windows OpenCASCADE is looking in the wrong location for these files (as of 7.7.1) -- just copy them # TODO - Don't hardcode the path - shutil.copytree(os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin")) + shutil.copytree( + os.path.join("win64", "vc14", "bind"), os.path.join("win64", "vc14", "bin") + ) self._cmake_install() os.chdir(cwd) @@ -799,9 +864,17 @@ def build_opencascade(self, _=None): # TODO - something is getting messed up in the CMake config output (note the quotes around 26812): for now just # drop the line entirely # set (OpenCASCADE_CXX_FLAGS "[...] /wd"26812" /MP /W4") - with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "r", encoding="utf-8") as f: + with open( + os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), + "r", + encoding="utf-8", + ) as f: occt_cmake_contents = f.readlines() - with open(os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), "w", encoding="utf-8") as f: + with open( + os.path.join(self.install_dir, "cmake", "OpenCASCADEConfig.cmake"), + "w", + encoding="utf-8", + ) as f: for line in occt_cmake_contents: if "OpenCASCADE_CXX_FLAGS" not in line: f.write(line + "\n") @@ -811,17 +884,19 @@ def build_netgen(self, _: None): if os.path.exists(os.path.join(self.install_dir, "share", "netgen")): print(" Not rebuilding netgen, it is already in the LibPack") return - extra_args = [f"-D CMAKE_FIND_ROOT_PATH={self.install_dir}", - "-D USE_SUPERBUILD=OFF", - "-D USE_GUI=OFF", - "-D USE_NATIVE_ARCH=OFF", - "-D USE_INTERNAL_TCL=OFF", - f"-D TCL_DIR={self.install_dir}", - f"-D TK_DIR={self.install_dir}", - "-D USE_OCC=On", - f"-D OpenCASCADE_ROOT={self.install_dir}", - f"-D USE_PYTHON=OFF", - f"-D CMAKE_CXX_FLAGS=-D_USE_MATH_DEFINES"] # To get M_PI on MSVC + extra_args = [ + f"-D CMAKE_FIND_ROOT_PATH={self.install_dir}", + "-D USE_SUPERBUILD=OFF", + "-D USE_GUI=OFF", + "-D USE_NATIVE_ARCH=OFF", + "-D USE_INTERNAL_TCL=OFF", + f"-D TCL_DIR={self.install_dir}", + f"-D TK_DIR={self.install_dir}", + "-D USE_OCC=On", + f"-D OpenCASCADE_ROOT={self.install_dir}", + f"-D USE_PYTHON=OFF", + f"-D CMAKE_CXX_FLAGS=-D_USE_MATH_DEFINES", + ] # To get M_PI on MSVC self._build_standard_cmake(extra_args=extra_args) def build_hdf5(self, _: None): @@ -843,18 +918,19 @@ def build_medfile(self, _: None): def build_gmsh(self, _: None): if self.skip_existing: - if os.path.exists( - os.path.join(self.install_dir, "bin", "gmsh" + to_exe())): + if os.path.exists(os.path.join(self.install_dir, "bin", "gmsh" + to_exe())): print(" Not rebuilding gmsh, it is already in the LibPack") return extra_args = [] if sys.platform.startswith("win32"): - extra_args = [f"-D CMAKE_LIBRARY_PATH={self.install_dir}/win64/vc14/lib", # TODO - Remove hardcoding - "-D ENABLE_OPENMP=No"] # Build fails if OpenMP is enabled + extra_args = [ + f"-D CMAKE_LIBRARY_PATH={self.install_dir}/win64/vc14/lib", # TODO - Remove hardcoding + "-D ENABLE_OPENMP=No", + ] # Build fails if OpenMP is enabled self._build_standard_cmake(extra_args) def build_pycxx(self, _: None): - """ PyCXX does not use a cMake-based build system """ + """PyCXX does not use a cMake-based build system""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "bin", "Lib", "site-packages", "CXX")): print(" Not rebuilding PyCXX, it is already in the LibPack") @@ -869,7 +945,7 @@ def build_pycxx(self, _: None): exit(1) def build_icu(self, _: None): - """ ICU does not use cMake, but has projects for various OSes """ + """ICU does not use cMake, but has projects for various OSes""" if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "unicode")): print(" Not rebuilding ICU, it is already in the LibPack") @@ -878,8 +954,15 @@ def build_icu(self, _: None): os.chdir(os.path.join("icu4c", "source")) if sys.platform.startswith("win32"): os.chdir("allinone") - args = [self.init_script, "&", "msbuild", f"/p:Configuration={str(self.mode).lower()}", - "/t:Build", "/p:SkipUWP=true", "allinone.sln"] + args = [ + self.init_script, + "&", + "msbuild", + f"/p:Configuration={str(self.mode).lower()}", + "/t:Build", + "/p:SkipUWP=true", + "allinone.sln", + ] subprocess.run(args, check=True, capture_output=True) os.chdir(os.path.join("..", "..")) bin_dir = os.path.join(self.install_dir, "bin") @@ -889,18 +972,18 @@ def build_icu(self, _: None): shutil.copytree(f"lib64", lib_dir, dirs_exist_ok=True) shutil.copytree(f"include", inc_dir, dirs_exist_ok=True) else: - raise NotImplemented( - "Non-Windows compilation of ICU is not implemented yet" - ) + raise NotImplemented("Non-Windows compilation of ICU is not implemented yet") def build_xercesc(self, _: None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "xercesc")): print(" Not rebuilding xerces-c, it is already in the LibPack") return - extra_args = [f"-D ICU_INCLUDE_DIR={self.install_dir}/include", - f"-D ICU_ROOT={self.install_dir}", - f"-D ICU_UC_DIR={self.install_dir}"] + extra_args = [ + f"-D ICU_INCLUDE_DIR={self.install_dir}/include", + f"-D ICU_ROOT={self.install_dir}", + f"-D ICU_UC_DIR={self.install_dir}", + ] self._build_standard_cmake(extra_args) def build_libfmt(self, _: None): @@ -925,7 +1008,7 @@ def build_yamlcpp(self, _: None): extra_args = ["-D YAML_BUILD_SHARED_LIBS=ON"] self._build_standard_cmake(extra_args) - def build_opencamlib(self, _:None): + def build_opencamlib(self, _: None): if self.skip_existing: if os.path.exists(os.path.join(self.install_dir, "include", "opencamlib")): print(" Not rebuilding opencamlib, it is already in the LibPack") diff --git a/create_libpack.py b/create_libpack.py index 28c2f5a..26ec24f 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -95,7 +95,9 @@ def create_libpack_dir(config: dict, mode: compile_all.BuildMode) -> str: backup_name = dirname + "-backup-" + "a" while os.path.exists(backup_name): if backup_name[-1] == "z": - print("You have too many old LibPack backup directories. Please delete some of them.") + print( + "You have too many old LibPack backup directories. Please delete some of them." + ) exit(1) backup_name = backup_name[:-1] + chr(ord(backup_name[-1]) + 1) @@ -179,7 +181,9 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): manifest_file = os.path.join(compile_all.libpack_dir(outer_config, mode_used), "manifest.json") with open(manifest_file, "w", encoding="utf-8") as f: f.write(json.dumps(outer_config["content"], indent=" ")) - version_file = os.path.join(compile_all.libpack_dir(outer_config, mode_used), "FREECAD_LIBPACK_VERSION") + version_file = os.path.join( + compile_all.libpack_dir(outer_config, mode_used), "FREECAD_LIBPACK_VERSION" + ) with open(version_file, "w", encoding="utf-8") as f: f.write(outer_config["LibPack-version"]) @@ -225,9 +229,7 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): help="I kow what I'm doing, don't ask me any questions", ) parser.add_argument("--7zip", help="Path to 7-zip executable", default=path_to_7zip) - parser.add_argument( - "--bison", help="Path to Bison executable", default=path_to_bison - ) + parser.add_argument("--bison", help="Path to Bison executable", default=path_to_bison) parser.add_argument("path-to-final-libpack-dir", nargs="?", default="./") args = vars(parser.parse_args()) @@ -237,7 +239,11 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): os.makedirs("working", exist_ok=True) os.chdir("working") - mode = compile_all.BuildMode.DEBUG if args["mode"].lower() == "debug" else compile_all.BuildMode.RELEASE + mode = ( + compile_all.BuildMode.DEBUG + if args["mode"].lower() == "debug" + else compile_all.BuildMode.RELEASE + ) if args["no_skip_existing_clone"]: dirname = compile_all.libpack_dir(config_dict, mode) if not os.path.exists(dirname): @@ -253,7 +259,7 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): config_dict, bison_path=path_to_bison, skip_existing=args["no_skip_existing_build"], - mode=mode + mode=mode, ) compiler.init_script = devel_init_script compiler.compile_all() diff --git a/generate_patch.py b/generate_patch.py index 8d361ff..cc7fd46 100644 --- a/generate_patch.py +++ b/generate_patch.py @@ -5,7 +5,9 @@ def print_usage(): - print("Generate a patchfile that can be used with the create_libpack.py script to patch source files") + print( + "Generate a patchfile that can be used with the create_libpack.py script to patch source files" + ) print("Usage: python generate_patch.py original_file corrected_file output_patch_file") diff --git a/path_cleaner.py b/path_cleaner.py index bdcc16e..d30ebc2 100644 --- a/path_cleaner.py +++ b/path_cleaner.py @@ -12,12 +12,12 @@ "env.bat", "draw.bat", "RELEASE.txt", - ] +] def delete_extraneous_files(base_path: str) -> None: """Delete each of the files listed above from the path specified in base_path. Failure to delete a file does not - constitute a fatal error.""" + constitute a fatal error.""" if not os.path.exists(base_path): raise RuntimeError(f"{base_path} does not exist") if not os.path.isdir(base_path): @@ -41,7 +41,7 @@ def remove_local_path_from_cmake_files(base_path: str) -> None: def remove_local_path_from_cmake_file(base_path: str, file_to_clean: str) -> None: - """ Modify a cMake file to remove base_path and replace it with ${CMAKE_CURRENT_SOURCE_DIR} -- WARNING: effectively + """Modify a cMake file to remove base_path and replace it with ${CMAKE_CURRENT_SOURCE_DIR} -- WARNING: effectively edits the file in-place, no backup is made.""" depth_string = create_depth_string(base_path, file_to_clean) with open(file_to_clean, "r", encoding="UTF-8") as f: @@ -63,4 +63,3 @@ def create_depth_string(base_path: str, file_to_clean: str) -> str: directories_in_base = len(base_path.split(os.path.sep)) num_steps_up = directories_to_file - directories_in_base return "../" * num_steps_up # For use in cMake, so always a forward slash here - diff --git a/test_compile_all.py b/test_compile_all.py index efb2066..31d5989 100644 --- a/test_compile_all.py +++ b/test_compile_all.py @@ -14,11 +14,11 @@ class TestCompileAll(unittest.TestCase): def setUp(self) -> None: super().setUp() - config = {"FreeCAD-version": "0.22", - "LibPack-version": "3.0.0", - "content": [ - {"name": "nonexistent"} - ]} + config = { + "FreeCAD-version": "0.22", + "LibPack-version": "3.0.0", + "content": [{"name": "nonexistent"}], + } self.compiler = compile_all.Compiler(config, compile_all.BuildMode.RELEASE, "bison_path") self.original_dir = os.getcwd() @@ -29,15 +29,13 @@ def tearDown(self) -> None: @patch("os.chdir") @patch("compile_all.Compiler.build_nonexistent") def test_compile_all_calls_build_function(self, nonexistent_mock: MagicMock, _): - config = {"content": [ - {"name": "nonexistent"} - ]} + config = {"content": [{"name": "nonexistent"}]} self.compiler.compile_all() nonexistent_mock.assert_called_once() @patch("subprocess.run") def test_get_python_version(self, run_mock: MagicMock): - """ Checking the Python version stores the Major and Minor components (but not the Patch) """ + """Checking the Python version stores the Major and Minor components (but not the Patch)""" # Arrange mock_result = MagicMock() diff --git a/test_create_libpack.py b/test_create_libpack.py index 9745522..6e84565 100644 --- a/test_create_libpack.py +++ b/test_create_libpack.py @@ -46,9 +46,7 @@ def test_with_directory_silent_deletes_dir(self, mock_print: MagicMock): self.assertFalse(os.path.exists(dir_to_delete)) @patch("builtins.input") - def test_with_directory_not_silent_asks_for_confirmation( - self, mock_input: MagicMock - ): + def test_with_directory_not_silent_asks_for_confirmation(self, mock_input: MagicMock): """When not in silent mode, the user is asked to confirm""" dir_to_delete = os.path.join(self.temp_dir.name, "existing_dir") os.mkdir(dir_to_delete) @@ -94,9 +92,7 @@ def test_json_is_loaded(self): def test_non_existent_file_prints_error(self, mock_print: MagicMock): """If a non-existent file is given, an error is printed (and exit() is called)""" with self.assertRaises(SystemExit): - create_libpack.load_config( - os.path.join(self.temp_dir.name, "no_such_file.json") - ) + create_libpack.load_config(os.path.join(self.temp_dir.name, "no_such_file.json")) mock_print.assert_called_once() @patch("builtins.print") @@ -213,16 +209,12 @@ def test_url_calls_download(self, download_mock: MagicMock): def test_download_creates_file(self, decompress_mock: MagicMock, _1, _2): with patch("builtins.open", mock_open()) as open_mock: create_libpack.download("make_this_dir", "https://some.url/test.7z") - open_mock.assert_called_once_with( - os.path.join("make_this_dir", "test.7z"), "wb" - ) + open_mock.assert_called_once_with(os.path.join("make_this_dir", "test.7z"), "wb") decompress_mock.assert_called_once_with("make_this_dir", "test.7z") @patch("os.chdir") @patch("subprocess.run") - def test_decompress_calls_subprocess( - self, run_mock: MagicMock, chdir_mock: MagicMock - ): + def test_decompress_calls_subprocess(self, run_mock: MagicMock, chdir_mock: MagicMock): create_libpack.decompress("path_to_file", "file_name") run_mock.assert_called_once() self.assertEqual(chdir_mock.call_count, 2) diff --git a/test_generate_patch.py b/test_generate_patch.py index 6ae0272..e99772b 100644 --- a/test_generate_patch.py +++ b/test_generate_patch.py @@ -10,6 +10,7 @@ """ Developer tests for the generate_patch module. """ + class TestGeneratePatch(unittest.TestCase): def setUp(self): @@ -43,8 +44,10 @@ def test_patch_generated(self): def test_run_loads_all_files(self): with patch("builtins.open", mock_open()) as open_mock: generate_patch.run("old", "new", "patch") - expected_calls = [unittest.mock.call("old","r",encoding="utf-8"), - unittest.mock.call("new","r",encoding="utf-8"), - unittest.mock.call("patch","w",encoding="utf-8")] + expected_calls = [ + unittest.mock.call("old", "r", encoding="utf-8"), + unittest.mock.call("new", "r", encoding="utf-8"), + unittest.mock.call("patch", "w", encoding="utf-8"), + ] for call in expected_calls: self.assertIn(call, open_mock.mock_calls) From cada717574a8a2cb4b1b22fe4124a5b9479e6f06 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 23 Jun 2024 20:02:58 -0500 Subject: [PATCH 26/27] Fixes for Boost compilation with MSVC 14.4 --- .gitignore | 2 +- Readme.md | 2 +- compile_all.py | 11 ++- config.json | 3 +- create_libpack.py | 4 + patches/boost-01-msvc_14_4_support.patch | 5 ++ path_cleaner.py | 25 +++++- test_path_cleaner.py | 100 +++++++++++++++++++++++ 8 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 patches/boost-01-msvc_14_4_support.patch create mode 100644 test_path_cleaner.py diff --git a/.gitignore b/.gitignore index 1af08ab..e2b18ab 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ working LibPack-* .venv venv -.idea \ No newline at end of file +.idea diff --git a/Readme.md b/Readme.md index 4a5193d..8ee89f7 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ This repository is to provide libraries needed to compile FreeCAD under Windows. -The LibPack is tested to work with [Microsoft Visual C++](https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B) (a.k.a. MSVC or VC). It should be possible to use other compilers like MinGW, however this is not tested. +The current LibPack, v3.0, is tested to work with [Microsoft Visual C++](https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B) (a.k.a. MSVC or VC) v14.4 (released mid-2024). It should be possible to use other compilers like MinGW, however this is not tested. This LibPack only supports FreeCAD compilation in Release or RelWithDebInfo mode. It may be possible to compile the LibPack in Debug mode, but changes will certainly be required (and patches are welcome!). In particular, the pip installation of Numpy will have to be adjusted to compile a debug version of Numpy, which will otherwise fail to load from a debug compilation of Python. For information how to use the LibPack to compile, see this Wiki page: https://wiki.freecadweb.org/Compile_on_Windows diff --git a/compile_all.py b/compile_all.py index 77d8088..a2b65ef 100644 --- a/compile_all.py +++ b/compile_all.py @@ -428,6 +428,7 @@ def build_boost(self, _=None): "b2", f"install", "address-model=64", + "architecture=x86", # TODO: Don't hardcode "link=static,shared", str(self.mode).lower(), "python-debugging=" + ("on" if self.mode == BuildMode.DEBUG else "off"), @@ -443,14 +444,16 @@ def build_boost(self, _=None): ) except subprocess.CalledProcessError as e: # Boost is too verbose in its output to be of much use un-processed. Dump it all to a file, and - # then print only the lines with the word "error" on them to stdout - print("Error: failed to build boost -- writing output to stdout.txt") + # then print only the lines with the word "error:" on them to stdout + print( + "Error: failed to build boost -- writing output to " + + os.path.join(os.path.curdir, "stdout.txt") + ) with open("stdout.txt", "w", encoding="utf-8") as f: f.write(e.stdout.decode("utf-8")) lines = e.stdout.decode("utf-8").split("\n") for line in lines: - if "error" in line.lower(): - # Lots of these lines are just files with the word 'error' in them, maybe there is a better filter? + if "error:" in line.lower(): print(line) exit(e.returncode) self._configure_boost_version() diff --git a/config.json b/config.json index 26c39ed..d173c7c 100644 --- a/config.json +++ b/config.json @@ -77,7 +77,8 @@ { "name":"boost", "git-repo":"https://github.com/boostorg/boost", - "git-ref":"boost-1.85.0" + "git-ref":"boost-1.85.0", + "patches": ["patches/boost-01-msvc_14_4_support.patch"] }, { "name":"coin", diff --git a/create_libpack.py b/create_libpack.py index 26ec24f..873fb04 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -32,6 +32,7 @@ import stat import subprocess from urllib.parse import urlparse +import path_cleaner try: import requests @@ -264,4 +265,7 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): compiler.init_script = devel_init_script compiler.compile_all() + path_cleaner.delete_extraneous_files(compile_all.libpack_dir(config_dict, mode)) + path_cleaner.remove_local_path_from_cmake_files(compile_all.libpack_dir(config_dict, mode)) + write_manifest(config_dict, mode) diff --git a/patches/boost-01-msvc_14_4_support.patch b/patches/boost-01-msvc_14_4_support.patch new file mode 100644 index 0000000..a5afa90 --- /dev/null +++ b/patches/boost-01-msvc_14_4_support.patch @@ -0,0 +1,5 @@ +@@@ tools/build/src/tools/msvc.jam @@@ +@@ -45127,32 +45127,375 @@ + if %5B ++ MATCH %22(14.4)%22 : $(version) %5D%0A %7B%0A if $(.debug-configuration)%0A %7B%0A ECHO %22notice: %5Bgenerate-setup-cmd%5D $(version) is 14.4%22 ;%0A %7D%0A parent = %5B path.native %5B path.join $(parent) %22..%5C%5C..%5C%5C..%5C%5C..%5C%5C..%5C%5CAuxiliary%5C%5CBuild%22 %5D %5D ;%0A %7D%0A else if %5B + MATCH %22(14.3)%22 diff --git a/path_cleaner.py b/path_cleaner.py index d30ebc2..5f4d797 100644 --- a/path_cleaner.py +++ b/path_cleaner.py @@ -27,6 +27,7 @@ def delete_extraneous_files(base_path: str) -> None: os.unlink(os.path.join(base_path, file)) except OSError as e: print(e) + print(" (continuing anyway...)") def remove_local_path_from_cmake_files(base_path: str) -> None: @@ -46,7 +47,24 @@ def remove_local_path_from_cmake_file(base_path: str, file_to_clean: str) -> Non depth_string = create_depth_string(base_path, file_to_clean) with open(file_to_clean, "r", encoding="UTF-8") as f: contents = f.read() - contents.replace(base_path, "${CMAKE_CURRENT_SOURCE_DIR}/" + depth_string) + + if base_path.endswith(os.path.sep): + base_path = base_path[: -len(os.path.sep)] + + # First, just replace the exact string we were given + contents = contents.replace( + base_path, "${CMAKE_CURRENT_SOURCE_DIR}/" + depth_string[:-1] + ) # Skip the final / + + # Most occurrences should NOT have been the exact string if we are on Windows, since cMake paths should always + # use forward slashes, so make sure to do that replacement as well + if os.pathsep != "/": + cmake_base_path = base_path.replace( + os.path.sep, "/" + ) # cMake paths should always use forward slash + contents = contents.replace( + cmake_base_path, "${CMAKE_CURRENT_SOURCE_DIR}/" + depth_string[:-1] + ) # Skip / with open(file_to_clean, "w", encoding="utf-8") as f: f.write(contents) @@ -55,9 +73,14 @@ def create_depth_string(base_path: str, file_to_clean: str) -> str: """Given a base path and a file, determine how many "../" must be appended to the file's containing directory to result in a path that resolves to base_path. Returns a string containing just some number of occurrences of "../" e.g. "../../../" to move up three levels from file_to_clean's containing folder.""" + + file_to_clean = os.path.normpath(file_to_clean) if not file_to_clean.startswith(base_path): raise RuntimeError(f"{file_to_clean} does not appear to be in {base_path}") + if base_path.endswith(os.path.sep): + base_path = base_path[: -len(os.path.sep)] + containing_directory = os.path.dirname(file_to_clean) directories_to_file = len(containing_directory.split(os.path.sep)) directories_in_base = len(base_path.split(os.path.sep)) diff --git a/test_path_cleaner.py b/test_path_cleaner.py new file mode 100644 index 0000000..e4b2908 --- /dev/null +++ b/test_path_cleaner.py @@ -0,0 +1,100 @@ +#!/bin/python3 +import os + +# SPDX-License-Identifier: LGPL-2.1-or-later + +import unittest +from unittest.mock import MagicMock, patch, mock_open + +import path_cleaner + + +class TestPathCleaner(unittest.TestCase): + """Tests the methods in path_cleaner.py""" + + def test_create_depth_string_simple(self): + """The Depth String method should return a cMake-style string consisting of dots and slashes (never + backslashes).""" + # Arrange + starts_with = os.path.join("some", "fake", "path") + fake_file_path = os.path.join(starts_with, "to", "a", "file.txt") + + # Act + result = path_cleaner.create_depth_string(starts_with, fake_file_path) + + # Assert + self.assertEqual( + "../../", result, "Expected a cMake-style path string going up two directories" + ) + + def test_create_depth_string_trailing_path_sep(self): + """Even if there is an extraneous trailing path separator on the base path, the method should return the + correct results.""" + # Arrange + starts_with = os.path.join("some", "fake", "path") + fake_file_path = os.path.join(starts_with, "to", "a", "file.txt") + + # Act + result = path_cleaner.create_depth_string(starts_with + os.path.sep, fake_file_path) + + # Assert + self.assertEqual( + "../../", result, "Expected a cMake-style path string going up two directories" + ) + + def test_create_depth_string_extraneous_slashes(self): + """Even if there are extraneous slashes in the path, it should still return the correct result""" + # Arrange + starts_with = os.path.join("some", "fake", "path") + fake_file_path = os.path.join(starts_with, "to", "a", "file.txt") + fake_file_path = fake_file_path.replace(os.path.sep, os.path.sep + os.path.sep) + + # Act + result = path_cleaner.create_depth_string(starts_with, fake_file_path) + + # Assert + self.assertEqual( + "../../", result, "Expected a cMake-style path string going up two directories" + ) + + def test_remove_local_path_from_cmake_file(self): + """Given a cMake file that contains some local paths, this should remove those local paths and convert them + into references relative to the file's location.""" + # Arrange + fake_cmake_data = ( + ' set(_BOOST_CMAKEDIR "Z:/FreeCAD/FreeCAD-LibPack-1.0.0-v3.0.0-Release/lib/cmake")\n' + ) + cleaned_data = ' set(_BOOST_CMAKEDIR "${CMAKE_CURRENT_SOURCE_DIR}/../../lib/cmake")\n' + + # Act + with patch("builtins.open", mock_open(read_data=fake_cmake_data)) as open_mock: + path_cleaner.remove_local_path_from_cmake_file( + "Z:\\FreeCAD\\FreeCAD-LibPack-1.0.0-v3.0.0-Release\\", + "Z:\\FreeCAD\\FreeCAD-LibPack-1.0.0-v3.0.0-Release\\lib\\cmake\\mock.cmake", + ) + + # Assert (still in the context manager, so we can query the mocked file) + open_mock().write.assert_called_with(cleaned_data) + + def test_remove_local_path_from_cmake_file_bad_path(self): + """There is at least one package (MEDfile) that puts in a Windows-style path into cMake, even though they + should not do so. Make sure we handle that.""" + # Arrange + fake_cmake_data = ( + 'SET(_hdf5_path "Z:\\FreeCAD\\FreeCAD-LibPack-1.0.0-v3.0.0-Release/share/cmake/")\n' + ) + cleaned_data = 'SET(_hdf5_path "${CMAKE_CURRENT_SOURCE_DIR}/../../share/cmake/")\n' + + # Act + with patch("builtins.open", mock_open(read_data=fake_cmake_data)) as open_mock: + path_cleaner.remove_local_path_from_cmake_file( + "Z:\\FreeCAD\\FreeCAD-LibPack-1.0.0-v3.0.0-Release\\", + "Z:\\FreeCAD\\FreeCAD-LibPack-1.0.0-v3.0.0-Release\\share\\cmake\\mock.cmake", + ) + + # Assert (still in the context manager, so we can query the mocked file) + open_mock().write.assert_called_with(cleaned_data) + + +if __name__ == "__main__": + unittest.main() From 404fcaf59ca289f4943c851a1cab6f64051b4f24 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 23 Jun 2024 21:37:36 -0500 Subject: [PATCH 27/27] Correct OpenCamLib compilation (their instructions are wrong) --- compile_all.py | 4 ++-- create_libpack.py | 7 +++++-- path_cleaner.py | 20 ++++++++++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/compile_all.py b/compile_all.py index a2b65ef..35b3e11 100644 --- a/compile_all.py +++ b/compile_all.py @@ -1013,8 +1013,8 @@ def build_yamlcpp(self, _: None): def build_opencamlib(self, _: None): if self.skip_existing: - if os.path.exists(os.path.join(self.install_dir, "include", "opencamlib")): + if os.path.exists(os.path.join(self.install_dir, "lib", "opencamlib", "ocl.lib")): print(" Not rebuilding opencamlib, it is already in the LibPack") return - extra_args = ["-D CXX_LIB=ON"] + extra_args = ["-D BUILD_CXX_LIB=ON -D BUILD_PY_LIB=ON -D BUILD_DOC=OFF"] self._build_standard_cmake(extra_args) diff --git a/create_libpack.py b/create_libpack.py index 873fb04..c25c424 100644 --- a/create_libpack.py +++ b/create_libpack.py @@ -265,7 +265,10 @@ def write_manifest(outer_config: dict, mode_used: compile_all.BuildMode): compiler.init_script = devel_init_script compiler.compile_all() - path_cleaner.delete_extraneous_files(compile_all.libpack_dir(config_dict, mode)) - path_cleaner.remove_local_path_from_cmake_files(compile_all.libpack_dir(config_dict, mode)) + # Final cleanup: delete extraneous files and remove local path references from the cMake files + base_path = compile_all.libpack_dir(config_dict, mode) + path_cleaner.delete_extraneous_files(base_path) + path_cleaner.remove_local_path_from_cmake_files(base_path) + path_cleaner.correct_opencascade_freetype_ref(base_path) write_manifest(config_dict, mode) diff --git a/path_cleaner.py b/path_cleaner.py index 5f4d797..4ff4c1a 100644 --- a/path_cleaner.py +++ b/path_cleaner.py @@ -26,8 +26,8 @@ def delete_extraneous_files(base_path: str) -> None: try: os.unlink(os.path.join(base_path, file)) except OSError as e: - print(e) - print(" (continuing anyway...)") + # If the file isn't there, that's as good as deleting it, right? + pass def remove_local_path_from_cmake_files(base_path: str) -> None: @@ -86,3 +86,19 @@ def create_depth_string(base_path: str, file_to_clean: str) -> str: directories_in_base = len(base_path.split(os.path.sep)) num_steps_up = directories_to_file - directories_in_base return "../" * num_steps_up # For use in cMake, so always a forward slash here + + +def correct_opencascade_freetype_ref(base_path: str): + """OpenCASCADE hardcodes the path to the freetype it was compiled against. The above code doesn't correct it to + the necessary path because of the way this variable is used within cMake. So just remove the path altogether and + rely on the rest of our configuration to find the correct one.""" + files_to_fix = ["OpenCASCADEDrawTargets.cmake", "OpenCASCADEVisualizationTargets.cmake"] + for fix in files_to_fix: + path = os.path.join(base_path, "cmake", fix) + with open(path, "r", encoding="utf-8") as f: + contents = f.read() + contents = contents.replace( + "${CMAKE_CURRENT_SOURCE_DIR}/../lib/freetype.lib", "freetype.lib" + ) + with open(path, "w", encoding="utf-8") as f: + f.write(contents)