diff options
| -rw-r--r-- | SConstruct | 185 | ||||
| -rw-r--r-- | scripts/build/cache.py | 127 | ||||
| -rw-r--r-- | scripts/build/glob_recursive.py | 15 | ||||
| -rw-r--r-- | scripts/build/option_handler.py | 39 |
4 files changed, 201 insertions, 165 deletions
@@ -4,14 +4,24 @@ import sys from glob import glob from pathlib import Path -# Neccessary to have our own build options without errors -SAVED_ARGUMENTS = ARGUMENTS.copy() -ARGUMENTS.pop('intermediate_delete', True) -ARGUMENTS.pop('progress', True) -ARGUMENTS.pop('verbose', True) -ARGUMENTS.pop('compiledb', True) +# Local +from scripts.build.option_handler import OptionsClass +from scripts.build.glob_recursive import GlobRecursive +from scripts.build.cache import show_progress -env = SConscript("godot-cpp/SConstruct") +opts = OptionsClass(ARGUMENTS) + +opts.Add(BoolVariable("compiledb", "Generate compilation DB (`compile_commands.json`) for external tools", False)) +opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False)) +opts.Add(BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True)) +opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True)) + + +# Needs Clone, else godot-cpp builds using our modified environment variables. eg: godot-cpp builds on C++20 +env = SConscript("godot-cpp/SConstruct").Clone() + +# Make LIBS into a list which is easier to deal with. +env["LIBS"] = [env["LIBS"]] # Require C++20 if env.get("is_msvc", False): @@ -19,167 +29,11 @@ if env.get("is_msvc", False): else: env.Replace(CXXFLAGS=["-std=c++20"]) -ARGUMENTS = SAVED_ARGUMENTS - # Custom options and profile flags. -customs = ["custom.py"] -profile = ARGUMENTS.get("profile", "") -if profile: - if os.path.isfile(profile): - customs.append(profile) - elif os.path.isfile(profile + ".py"): - customs.append(profile + ".py") -opts = Variables(customs, ARGUMENTS) - -opts.Add(BoolVariable("compiledb", "Generate compilation DB (`compile_commands.json`) for external tools", False)) -opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False)) -opts.Add( - BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True) -) -opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True)) - -opts.Update(env) +opts.Make(["custom.py"]) +opts.Finalize(env) Help(opts.GenerateHelpText(env)) -def GlobRecursive(pattern, nodes=['.']): - import SCons - results = [] - for node in nodes: - nnodes = [] - for f in Glob(str(node) + '/*', source=True): - if type(f) is SCons.Node.FS.Dir: - nnodes.append(f) - results += GlobRecursive(pattern, nnodes) - results += Glob(str(node) + '/' + pattern, source=True) - return results - -# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172 -def show_progress(env): - import sys - import glob - from SCons.Script import Progress, Command, AlwaysBuild - - screen = sys.stdout - # Progress reporting is not available in non-TTY environments since it - # messes with the output (for example, when writing to a file) - show_progress = env["progress"] and sys.stdout.isatty() - node_count = 0 - node_count_max = 0 - node_count_interval = 1 - node_count_fname = str(env.Dir("#")) + "/.scons_node_count" - - import time, math - - class cache_progress: - # The default is 1 GB cache and 12 hours half life - def __init__(self, path=None, limit=1073741824, half_life=43200): - self.path = path - self.limit = limit - self.exponent_scale = math.log(2) / half_life - if env["verbose"] and path != None: - screen.write( - "Current cache limit is {} (used: {})\n".format( - self.convert_size(limit), self.convert_size(self.get_size(path)) - ) - ) - self.delete(self.file_list()) - - def __call__(self, node, *args, **kw): - nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress - if show_progress: - # Print the progress percentage - node_count += node_count_interval - if node_count_max > 0 and node_count <= node_count_max: - screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max)) - screen.flush() - elif node_count_max > 0 and node_count > node_count_max: - screen.write("\r[100%] ") - screen.flush() - else: - screen.write("\r[Initial build] ") - screen.flush() - - def delete(self, files): - if len(files) == 0: - return - if env["verbose"]: - # Utter something - screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file")) - [os.remove(f) for f in files] - - def file_list(self): - if self.path is None: - # Nothing to do - return [] - # Gather a list of (filename, (size, atime)) within the - # cache directory - file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))] - if file_stat == []: - # Nothing to do - return [] - # Weight the cache files by size (assumed to be roughly - # proportional to the recompilation time) times an exponential - # decay since the ctime, and return a list with the entries - # (filename, size, weight). - current_time = time.time() - file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat] - # Sort by the most recently accessed files (most sensible to keep) first - file_stat.sort(key=lambda x: x[2]) - # Search for the first entry where the storage limit is - # reached - sum, mark = 0, None - for i, x in enumerate(file_stat): - sum += x[1] - if sum > self.limit: - mark = i - break - if mark is None: - return [] - else: - return [x[0] for x in file_stat[mark:]] - - def convert_size(self, size_bytes): - if size_bytes == 0: - return "0 bytes" - size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - i = int(math.floor(math.log(size_bytes, 1024))) - p = math.pow(1024, i) - s = round(size_bytes / p, 2) - return "%s %s" % (int(s) if i == 0 else s, size_name[i]) - - def get_size(self, start_path="."): - total_size = 0 - for dirpath, dirnames, filenames in os.walk(start_path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - - def progress_finish(target, source, env): - nonlocal node_count, progressor - try: - with open(node_count_fname, "w") as f: - f.write("%d\n" % node_count) - progressor.delete(progressor.file_list()) - except Exception: - pass - - try: - with open(node_count_fname) as f: - node_count_max = int(f.readline()) - except Exception: - pass - - cache_directory = os.environ.get("SCONS_CACHE") - # Simple cache pruning, attached to SCons' progress callback. Trim the - # cache directory to a size not larger than cache_limit. - cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - progressor = cache_progress(cache_directory, cache_limit) - Progress(progressor, interval=node_count_interval) - - progress_finish_command = Command("progress_finish", [], progress_finish) - AlwaysBuild(progress_finish_command) - scons_cache_path = os.environ.get("SCONS_CACHE") if scons_cache_path != None: CacheDir(scons_cache_path) @@ -210,6 +64,7 @@ if env["compiledb"]: paths = ["extension/src/", "extension/deps/openvic-simulation/src/"] env.Append(CPPPATH=paths) sources = GlobRecursive("*.cpp", paths) +env.extension_sources = sources # Remove unassociated intermediate binary files if allowed, usually the result of a renamed or deleted source file if env["intermediate_delete"]: diff --git a/scripts/build/cache.py b/scripts/build/cache.py new file mode 100644 index 0000000..d48b0e0 --- /dev/null +++ b/scripts/build/cache.py @@ -0,0 +1,127 @@ +# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172 +def show_progress(env): + import os + import sys + import glob + from SCons.Script import Progress, Command, AlwaysBuild + + screen = sys.stdout + # Progress reporting is not available in non-TTY environments since it + # messes with the output (for example, when writing to a file) + show_progress = env["progress"] and sys.stdout.isatty() + node_count = 0 + node_count_max = 0 + node_count_interval = 1 + node_count_fname = str(env.Dir("#")) + "/.scons_node_count" + + import time, math + + class cache_progress: + # The default is 1 GB cache and 12 hours half life + def __init__(self, path=None, limit=1073741824, half_life=43200): + self.path = path + self.limit = limit + self.exponent_scale = math.log(2) / half_life + if env["verbose"] and path != None: + screen.write( + "Current cache limit is {} (used: {})\n".format( + self.convert_size(limit), self.convert_size(self.get_size(path)) + ) + ) + self.delete(self.file_list()) + + def __call__(self, node, *args, **kw): + nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress + if show_progress: + # Print the progress percentage + node_count += node_count_interval + if node_count_max > 0 and node_count <= node_count_max: + screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max)) + screen.flush() + elif node_count_max > 0 and node_count > node_count_max: + screen.write("\r[100%] ") + screen.flush() + else: + screen.write("\r[Initial build] ") + screen.flush() + + def delete(self, files): + if len(files) == 0: + return + if env["verbose"]: + # Utter something + screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file")) + [os.remove(f) for f in files] + + def file_list(self): + if self.path is None: + # Nothing to do + return [] + # Gather a list of (filename, (size, atime)) within the + # cache directory + file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))] + if file_stat == []: + # Nothing to do + return [] + # Weight the cache files by size (assumed to be roughly + # proportional to the recompilation time) times an exponential + # decay since the ctime, and return a list with the entries + # (filename, size, weight). + current_time = time.time() + file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat] + # Sort by the most recently accessed files (most sensible to keep) first + file_stat.sort(key=lambda x: x[2]) + # Search for the first entry where the storage limit is + # reached + sum, mark = 0, None + for i, x in enumerate(file_stat): + sum += x[1] + if sum > self.limit: + mark = i + break + if mark is None: + return [] + else: + return [x[0] for x in file_stat[mark:]] + + def convert_size(self, size_bytes): + if size_bytes == 0: + return "0 bytes" + size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (int(s) if i == 0 else s, size_name[i]) + + def get_size(self, start_path="."): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(start_path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + + def progress_finish(target, source, env): + nonlocal node_count, progressor + try: + with open(node_count_fname, "w") as f: + f.write("%d\n" % node_count) + progressor.delete(progressor.file_list()) + except Exception: + pass + + try: + with open(node_count_fname) as f: + node_count_max = int(f.readline()) + except Exception: + pass + + cache_directory = os.environ.get("SCONS_CACHE") + # Simple cache pruning, attached to SCons' progress callback. Trim the + # cache directory to a size not larger than cache_limit. + cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 + progressor = cache_progress(cache_directory, cache_limit) + Progress(progressor, interval=node_count_interval) + + progress_finish_command = Command("progress_finish", [], progress_finish) + AlwaysBuild(progress_finish_command) diff --git a/scripts/build/glob_recursive.py b/scripts/build/glob_recursive.py new file mode 100644 index 0000000..db6eb80 --- /dev/null +++ b/scripts/build/glob_recursive.py @@ -0,0 +1,15 @@ +def GlobRecursive(pattern, nodes=['.']): + import SCons + import glob + fs = SCons.Node.FS.get_default_fs() + Glob = fs.Glob + + results = [] + for node in nodes: + nnodes = [] + for f in Glob(str(node) + '/*', source=True): + if type(f) is SCons.Node.FS.Dir: + nnodes.append(f) + results += GlobRecursive(pattern, nnodes) + results += Glob(str(node) + '/' + pattern, source=True) + return results diff --git a/scripts/build/option_handler.py b/scripts/build/option_handler.py new file mode 100644 index 0000000..3cebc1a --- /dev/null +++ b/scripts/build/option_handler.py @@ -0,0 +1,39 @@ +from typing import Tuple, Iterable +from SCons.Variables import Variables + +class OptionsClass: + def __init__(self, args): + self.opts = None + self.opt_list = [] + self.args = args + self.saved_args = args.copy() + + def Add(self, variableOrKey, *argv, **kwarg): + + self.opt_list.append([variableOrKey, argv, kwarg]) + # Neccessary to have our own build options without errors + if isinstance(variableOrKey, str): + self.args.pop(variableOrKey, True) + else: + self.args.pop(variableOrKey[0], True) + + def Make(self, customs : Iterable[str]): + self.args = self.saved_args + profile = self.args.get("profile", "") + if profile: + if os.path.isfile(profile): + customs.append(profile) + elif os.path.isfile(profile + ".py"): + customs.append(profile + ".py") + self.opts = Variables(customs, self.args) + for opt in self.opt_list: + if opt[1] == None and opt[2] == None: + self.opts.Add(opt[0]) + else: + self.opts.Add(opt[0], *opt[1], **opt[2]) + + def Finalize(self, env): + self.opts.Update(env) + + def GenerateHelpText(self, env): + return self.opts.GenerateHelpText(env) |