diff options
| -rw-r--r-- | .github/workflows/builds.yml | 141 | ||||
| -rw-r--r-- | .gitmodules | 3 | ||||
| -rw-r--r-- | SConstruct | 216 | ||||
| -rw-r--r-- | deps/SCsub | 12 | ||||
| m--------- | deps/openvic-dataloader | 0 | ||||
| m--------- | scripts | 0 | ||||
| -rw-r--r-- | scripts/build/cache.py | 127 | ||||
| -rw-r--r-- | scripts/build/glob_recursive.py | 15 | ||||
| -rw-r--r-- | scripts/build/option_handler.py | 39 |
9 files changed, 167 insertions, 386 deletions
diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml new file mode 100644 index 0000000..71c4054 --- /dev/null +++ b/.github/workflows/builds.yml @@ -0,0 +1,141 @@ +name: Builds + +on: [push, pull_request] + +env: + GH_BASE_BRANCH: master + +concurrency: + group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-macos + cancel-in-progress: true + +jobs: + build: + runs-on: ${{matrix.os}} + name: ${{matrix.name}} + permissions: write-all + strategy: + fail-fast: false + matrix: + include: + - identifier: windows-debug + os: windows-latest + name: 🏁 Windows Debug + target: template_debug + platform: windows + arch: x86_64 + - identifier: windows-release + os: windows-latest + name: 🏁 Windows Release + target: template_release + platform: windows + arch: x86_64 + - identifier: macos-debug + os: macos-latest + name: 🍎 macOS (universal) Debug + target: template_debug + platform: macos + arch: universal + - identifier: macos-release + os: macos-latest + name: 🍎 macOS (universal) Release + target: template_release + platform: macos + arch: universal + - identifier: linux-debug + os: ubuntu-latest + name: 🐧 Linux Debug + runner: ubuntu-20.04 + target: template_debug + platform: linux + arch: x86_64 + - identifier: linux-release + os: ubuntu-latest + name: 🐧 Linux Release + runner: ubuntu-20.04 + target: template_release + platform: linux + arch: x86_64 + + steps: + - name: Checkout project + uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Setup build cache + uses: OpenVicProject/openvic-cache@master + with: + cache-name: ${{ matrix.identifier }} + base-branch: ${{ env.GH_BASE_BRANCH }} + continue-on-error: true + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Set up SCons + shell: bash + run: | + python -c "import sys; print(sys.version)" + python -m pip install scons + scons --version + + - name: Linux dependencies + if: ${{ matrix.platform == 'linux' }} + run: | + sudo apt-get update -qq + sudo apt-get install -qqq build-essential pkg-config + g++ --version + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12 + sudo update-alternatives --set g++ /usr/bin/g++-12 + g++ --version + + - name: Setup MinGW for Windows/MinGW build + if: ${{ matrix.platform == 'windows' }} + uses: OpenVicProject/mingw-cache@master + + - name: Compile with SCons + uses: OpenVicProject/openvic-build@master + with: + platform: ${{ matrix.platform }} + target: ${{ matrix.target }} + sconsflags: arch=${{ matrix.arch }} build_ovsim_library=yes + + - name: Delete compilation files + if: ${{ matrix.platform == 'windows' }} + run: | + Remove-Item bin/* -Include *.exp,*.pdb -Force + + - name: Upload library artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ github.event.repository.name }}-library + path: | + ${{ github.workspace }}/bin/libopenvic-simulation.* + + - name: Upload executable artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ github.event.repository.name }}-executable + path: | + ${{ github.workspace }}/bin/openvic-simulation.headless.* + + - name: Archive Release + uses: thedoctor0/zip-release@0.7.1 + with: + type: "zip" + filename: "../../../libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip" + directory: "${{ github.workspace }}/bin/" + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + - name: Create and upload asset + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: "libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip" + omitNameDuringUpdate: true + omitBodyDuringUpdate: true + token: ${{ secrets.GITHUB_TOKEN }} + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') diff --git a/.gitmodules b/.gitmodules index df89ab5..1728f7a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "deps/openvic-dataloader"] path = deps/openvic-dataloader url = https://github.com/OpenVicProject/OpenVic-Dataloader +[submodule "scripts"] + path = scripts + url = git@github.com:OpenVicProject/scripts @@ -1,219 +1,25 @@ #!/usr/bin/env python -# This file is heavily based on https://github.com/godotengine/godot-cpp/blob/8155f35b29b4b08bc54b2eb0c57e1e9effe9f093/SConstruct import os import platform import sys -import importlib import SCons -# Local -from scripts.build.option_handler import OptionsClass -from scripts.build.glob_recursive import GlobRecursive -from scripts.build.cache import show_progress - -# Try to detect the host platform automatically. -# This is used if no `platform` argument is passed -if sys.platform.startswith("linux"): - default_platform = "linux" -elif sys.platform == "darwin": - default_platform = "macos" -elif sys.platform == "win32" or sys.platform == "msys": - default_platform = "windows" -elif ARGUMENTS.get("platform", ""): - default_platform = ARGUMENTS.get("platform") -else: - raise ValueError("Could not detect platform automatically, please specify with platform=<platform>") - -is_standalone = SCons.Script.sconscript_reading == 1 - BINDIR = "bin" -TOOLPATH = ["deps/openvic-dataloader/tools"] -try: - Import("env") - old_env = env - env = old_env.Clone() -except: - # Default tools with no platform defaults to gnu toolchain. - # We apply platform specific toolchains via our custom tools. - env = Environment(tools=["default"], PLATFORM="") - old_env = env +env = SConscript("scripts/SConstruct") env.PrependENVPath("PATH", os.getenv("PATH")) -# Default num_jobs to local cpu count if not user specified. -# SCons has a peculiarity where user-specified options won't be overridden -# by SetOption, so we can rely on this to know if we should use our default. -initial_num_jobs = env.GetOption("num_jobs") -altered_num_jobs = initial_num_jobs + 1 -env.SetOption("num_jobs", altered_num_jobs) -if env.GetOption("num_jobs") == altered_num_jobs: - cpu_count = os.cpu_count() - if cpu_count is None: - print("Couldn't auto-detect CPU count to configure build parallelism. Specify it with the -j argument.") - else: - safer_cpu_count = cpu_count if cpu_count <= 4 else cpu_count - 1 - print( - "Auto-detected %d CPU cores available for build parallelism. Using %d cores by default. You can override it with the -j argument." - % (cpu_count, safer_cpu_count) - ) - env.SetOption("num_jobs", safer_cpu_count) - -opts = OptionsClass(ARGUMENTS) - -platforms = ("linux", "macos", "windows", "android", "ios", "javascript") -unsupported_known_platforms = ("android", "ios", "javascript") -opts.Add( - EnumVariable( - key="platform", - help="Target platform", - default=env.get("platform", default_platform), - allowed_values=platforms, - ignorecase=2, - ) -) - -opts.Add( - EnumVariable( - key="target", - help="Compilation target", - default=env.get("target", "template_debug"), - allowed_values=("editor", "template_release", "template_debug"), - ) -) - -opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not is_standalone))) -opts.Add( - EnumVariable( - key="precision", - help="Set the floating-point precision level", - default=env.get("precision", "single"), - allowed_values=("single", "double"), - ) -) - -# Add platform options -tools = {} -for pl in set(platforms) - set(unsupported_known_platforms): - tool = Tool(pl, toolpath=TOOLPATH) - if hasattr(tool, "options"): - tool.options(opts) - tools[pl] = tool - -# CPU architecture options. -architecture_array = ["", "universal", "x86_32", "x86_64", "arm32", "arm64", "rv64", "ppc32", "ppc64", "wasm32"] -architecture_aliases = { - "x64": "x86_64", - "amd64": "x86_64", - "armv7": "arm32", - "armv8": "arm64", - "arm64v8": "arm64", - "aarch64": "arm64", - "rv": "rv64", - "riscv": "rv64", - "riscv64": "rv64", - "ppcle": "ppc32", - "ppc": "ppc32", - "ppc64le": "ppc64", -} -opts.Add( - EnumVariable( - key="arch", - help="CPU architecture", - default=env.get("arch", ""), - allowed_values=architecture_array, - map=architecture_aliases, - ) -) - -opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", is_standalone)) - -opts.Add(BoolVariable("compiledb", "Generate compilation DB (`compile_commands.json`) for external tools", False)) -opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False)) -opts.Add(BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True)) -opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True)) - -# Targets flags tool (optimizations, debug symbols) -target_tool = Tool("targets", toolpath=TOOLPATH) -target_tool.options(opts) - -# Custom options and profile flags. -opts.Make(["custom.py"]) -opts.Finalize(env) -Help(opts.GenerateHelpText(env)) - -if env["platform"] in unsupported_known_platforms: - print("Unsupported platform: " + env["platform"]+". Only supports " + ", ".join(set(platforms) - set(unsupported_known_platforms))) - Exit() - -# Process CPU architecture argument. -if env["arch"] == "": - # No architecture specified. Default to arm64 if building for Android, - # universal if building for macOS or iOS, wasm32 if building for web, - # otherwise default to the host architecture. - if env["platform"] in ["macos", "ios"]: - env["arch"] = "universal" - elif env["platform"] == "android": - env["arch"] = "arm64" - elif env["platform"] == "javascript": - env["arch"] = "wasm32" - else: - host_machine = platform.machine().lower() - if host_machine in architecture_array: - env["arch"] = host_machine - elif host_machine in architecture_aliases.keys(): - env["arch"] = architecture_aliases[host_machine] - elif "86" in host_machine: - # Catches x86, i386, i486, i586, i686, etc. - env["arch"] = "x86_32" - else: - print("Unsupported CPU architecture: " + host_machine) - Exit() - -tool = Tool(env["platform"], toolpath=TOOLPATH) - -if tool is None or not tool.exists(env): - raise ValueError("Required toolchain not found for platform " + env["platform"]) - -tool.generate(env) -target_tool.generate(env) - -print("Building for architecture " + env["arch"] + " on platform " + env["platform"]) - -# Require C++20 -if env.get("is_msvc", False): - env.Append(CXXFLAGS=["/std:c++20"]) -else: - env.Append(CXXFLAGS=["-std=c++20"]) - -if env["precision"] == "double": - env.Append(CPPDEFINES=["REAL_T_IS_DOUBLE"]) - -scons_cache_path = os.environ.get("SCONS_CACHE") -if scons_cache_path != None: - CacheDir(scons_cache_path) - print("Scons cache enabled... (path: '" + scons_cache_path + "')") - -if env["compiledb"]: - # Generating the compilation DB (`compile_commands.json`) requires SCons 4.0.0 or later. - from SCons import __version__ as scons_raw_version - - scons_ver = env._get_major_minor_revision(scons_raw_version) - - if scons_ver < (4, 0, 0): - print("The `compiledb=yes` option requires SCons 4.0 or later, but your version is %s." % scons_raw_version) - Exit(255) +opts = env.SetupOptions() - env.Tool("compilation_db") - env.Alias("compiledb", env.CompilationDatabase()) +opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not env.is_standalone))) +opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", env.is_standalone)) -ovdl_env = SConscript("deps/openvic-dataloader/SConstruct") +env.FinalizeOptions() -env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"]) -env.Append(LIBS=ovdl_env.openvic_dataloader["LIBS"]) -env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"]) +SConscript("deps/SCsub") env.openvic_simulation = {} @@ -229,7 +35,7 @@ env.openvic_simulation = {} source_path = "src/openvic-simulation" include_path = "src" env.Append(CPPPATH=[[env.Dir(p) for p in [source_path, include_path]]]) -sources = GlobRecursive("*.cpp", [source_path]) +sources = env.GlobRecursive("*.cpp", [source_path]) env.simulation_sources = sources suffix = ".{}.{}".format(env["platform"], env["target"]) @@ -251,11 +57,11 @@ if env["build_ovsim_library"]: Default(library) env.Append(LIBPATH=[env.Dir(BINDIR)]) - env.Append(LIBS=[library_name]) + env.Prepend(LIBS=[library_name]) env.openvic_simulation["LIBPATH"] = env["LIBPATH"] env.openvic_simulation["LIBS"] = env["LIBS"] - env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + ovdl_env.openvic_dataloader["INCPATH"] + env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + env.openvic_dataloader["INCPATH"] headless_program = None env["PROGSUFFIX"] = suffix + env["PROGSUFFIX"] @@ -266,7 +72,7 @@ if env["build_ovsim_headless"]: headless_path = ["src/headless"] headless_env.Append(CPPDEFINES=["OPENVIC_SIM_HEADLESS"]) headless_env.Append(CPPPATH=[headless_env.Dir(headless_path)]) - headless_env.headless_sources = GlobRecursive("*.cpp", headless_path) + headless_env.headless_sources = env.GlobRecursive("*.cpp", headless_path) if not env["build_ovsim_library"]: headless_env.headless_sources += sources headless_program = headless_env.Program( @@ -279,6 +85,6 @@ if env["build_ovsim_headless"]: if "env" in locals(): # FIXME: This method mixes both cosmetic progress stuff and cache handling... - show_progress(env) + env.show_progress(env) Return("env") diff --git a/deps/SCsub b/deps/SCsub new file mode 100644 index 0000000..7c945e6 --- /dev/null +++ b/deps/SCsub @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +Import("env") + +def build_openvic_dataloader(env): + ovdl_env = SConscript("openvic-dataloader/SConstruct") + env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"]) + env.Prepend(LIBS=ovdl_env.openvic_dataloader["LIBS"]) + env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"]) + env.openvic_dataloader = ovdl_env.openvic_dataloader + +build_openvic_dataloader(env)
\ No newline at end of file diff --git a/deps/openvic-dataloader b/deps/openvic-dataloader -Subproject 977661f6f4301be19fa64abfc6cda5040c3899b +Subproject 706c2413c6b390a18389414493218c21a59f495 diff --git a/scripts b/scripts new file mode 160000 +Subproject 71803e491af7174f536e72f76dfb74708b4b84f diff --git a/scripts/build/cache.py b/scripts/build/cache.py deleted file mode 100644 index d48b0e0..0000000 --- a/scripts/build/cache.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172 -def show_progress(env): - import os - import sys - import glob - from SCons.Script import Progress, Command, AlwaysBuild - - screen = sys.stdout - # Progress reporting is not available in non-TTY environments since it - # messes with the output (for example, when writing to a file) - show_progress = env["progress"] and sys.stdout.isatty() - node_count = 0 - node_count_max = 0 - node_count_interval = 1 - node_count_fname = str(env.Dir("#")) + "/.scons_node_count" - - import time, math - - class cache_progress: - # The default is 1 GB cache and 12 hours half life - def __init__(self, path=None, limit=1073741824, half_life=43200): - self.path = path - self.limit = limit - self.exponent_scale = math.log(2) / half_life - if env["verbose"] and path != None: - screen.write( - "Current cache limit is {} (used: {})\n".format( - self.convert_size(limit), self.convert_size(self.get_size(path)) - ) - ) - self.delete(self.file_list()) - - def __call__(self, node, *args, **kw): - nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress - if show_progress: - # Print the progress percentage - node_count += node_count_interval - if node_count_max > 0 and node_count <= node_count_max: - screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max)) - screen.flush() - elif node_count_max > 0 and node_count > node_count_max: - screen.write("\r[100%] ") - screen.flush() - else: - screen.write("\r[Initial build] ") - screen.flush() - - def delete(self, files): - if len(files) == 0: - return - if env["verbose"]: - # Utter something - screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file")) - [os.remove(f) for f in files] - - def file_list(self): - if self.path is None: - # Nothing to do - return [] - # Gather a list of (filename, (size, atime)) within the - # cache directory - file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))] - if file_stat == []: - # Nothing to do - return [] - # Weight the cache files by size (assumed to be roughly - # proportional to the recompilation time) times an exponential - # decay since the ctime, and return a list with the entries - # (filename, size, weight). - current_time = time.time() - file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat] - # Sort by the most recently accessed files (most sensible to keep) first - file_stat.sort(key=lambda x: x[2]) - # Search for the first entry where the storage limit is - # reached - sum, mark = 0, None - for i, x in enumerate(file_stat): - sum += x[1] - if sum > self.limit: - mark = i - break - if mark is None: - return [] - else: - return [x[0] for x in file_stat[mark:]] - - def convert_size(self, size_bytes): - if size_bytes == 0: - return "0 bytes" - size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - i = int(math.floor(math.log(size_bytes, 1024))) - p = math.pow(1024, i) - s = round(size_bytes / p, 2) - return "%s %s" % (int(s) if i == 0 else s, size_name[i]) - - def get_size(self, start_path="."): - total_size = 0 - for dirpath, dirnames, filenames in os.walk(start_path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - - def progress_finish(target, source, env): - nonlocal node_count, progressor - try: - with open(node_count_fname, "w") as f: - f.write("%d\n" % node_count) - progressor.delete(progressor.file_list()) - except Exception: - pass - - try: - with open(node_count_fname) as f: - node_count_max = int(f.readline()) - except Exception: - pass - - cache_directory = os.environ.get("SCONS_CACHE") - # Simple cache pruning, attached to SCons' progress callback. Trim the - # cache directory to a size not larger than cache_limit. - cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - progressor = cache_progress(cache_directory, cache_limit) - Progress(progressor, interval=node_count_interval) - - progress_finish_command = Command("progress_finish", [], progress_finish) - AlwaysBuild(progress_finish_command) diff --git a/scripts/build/glob_recursive.py b/scripts/build/glob_recursive.py deleted file mode 100644 index db6eb80..0000000 --- a/scripts/build/glob_recursive.py +++ /dev/null @@ -1,15 +0,0 @@ -def GlobRecursive(pattern, nodes=['.']): - import SCons - import glob - fs = SCons.Node.FS.get_default_fs() - Glob = fs.Glob - - results = [] - for node in nodes: - nnodes = [] - for f in Glob(str(node) + '/*', source=True): - if type(f) is SCons.Node.FS.Dir: - nnodes.append(f) - results += GlobRecursive(pattern, nnodes) - results += Glob(str(node) + '/' + pattern, source=True) - return results diff --git a/scripts/build/option_handler.py b/scripts/build/option_handler.py deleted file mode 100644 index 3cebc1a..0000000 --- a/scripts/build/option_handler.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Tuple, Iterable -from SCons.Variables import Variables - -class OptionsClass: - def __init__(self, args): - self.opts = None - self.opt_list = [] - self.args = args - self.saved_args = args.copy() - - def Add(self, variableOrKey, *argv, **kwarg): - - self.opt_list.append([variableOrKey, argv, kwarg]) - # Neccessary to have our own build options without errors - if isinstance(variableOrKey, str): - self.args.pop(variableOrKey, True) - else: - self.args.pop(variableOrKey[0], True) - - def Make(self, customs : Iterable[str]): - self.args = self.saved_args - profile = self.args.get("profile", "") - if profile: - if os.path.isfile(profile): - customs.append(profile) - elif os.path.isfile(profile + ".py"): - customs.append(profile + ".py") - self.opts = Variables(customs, self.args) - for opt in self.opt_list: - if opt[1] == None and opt[2] == None: - self.opts.Add(opt[0]) - else: - self.opts.Add(opt[0], *opt[1], **opt[2]) - - def Finalize(self, env): - self.opts.Update(env) - - def GenerateHelpText(self, env): - return self.opts.GenerateHelpText(env) |