aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author Spartan322 <Megacake1234@gmail.com>2023-09-23 23:33:57 +0200
committer Spartan322 <Megacake1234@gmail.com>2023-09-24 01:13:20 +0200
commit7b8415400fe90325905771a0a21dc1edb66b86c2 (patch)
treeb5ff74f9f5de6c8bda867cd4667479f5bf29c7e1
parent005a8026bb424779a146e00cc48621ff1d72b807 (diff)
Move scripts to separate reusable repos
Add deps/SCsub to simplify SConstruct Added github build workflow Fix linker problems
-rw-r--r--.github/workflows/builds.yml141
-rw-r--r--.gitmodules3
-rw-r--r--SConstruct216
-rw-r--r--deps/SCsub12
m---------deps/openvic-dataloader0
m---------scripts0
-rw-r--r--scripts/build/cache.py127
-rw-r--r--scripts/build/glob_recursive.py15
-rw-r--r--scripts/build/option_handler.py39
9 files changed, 167 insertions, 386 deletions
diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
new file mode 100644
index 0000000..71c4054
--- /dev/null
+++ b/.github/workflows/builds.yml
@@ -0,0 +1,141 @@
+name: Builds
+
+on: [push, pull_request]
+
+env:
+ GH_BASE_BRANCH: master
+
+concurrency:
+ group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-macos
+ cancel-in-progress: true
+
+jobs:
+ build:
+ runs-on: ${{matrix.os}}
+ name: ${{matrix.name}}
+ permissions: write-all
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - identifier: windows-debug
+ os: windows-latest
+ name: 🏁 Windows Debug
+ target: template_debug
+ platform: windows
+ arch: x86_64
+ - identifier: windows-release
+ os: windows-latest
+ name: 🏁 Windows Release
+ target: template_release
+ platform: windows
+ arch: x86_64
+ - identifier: macos-debug
+ os: macos-latest
+ name: 🍎 macOS (universal) Debug
+ target: template_debug
+ platform: macos
+ arch: universal
+ - identifier: macos-release
+ os: macos-latest
+ name: 🍎 macOS (universal) Release
+ target: template_release
+ platform: macos
+ arch: universal
+ - identifier: linux-debug
+ os: ubuntu-latest
+ name: 🐧 Linux Debug
+ runner: ubuntu-20.04
+ target: template_debug
+ platform: linux
+ arch: x86_64
+ - identifier: linux-release
+ os: ubuntu-latest
+ name: 🐧 Linux Release
+ runner: ubuntu-20.04
+ target: template_release
+ platform: linux
+ arch: x86_64
+
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v3.3.0
+ with:
+ submodules: recursive
+
+ - name: Setup build cache
+ uses: OpenVicProject/openvic-cache@master
+ with:
+ cache-name: ${{ matrix.identifier }}
+ base-branch: ${{ env.GH_BASE_BRANCH }}
+ continue-on-error: true
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.x"
+
+ - name: Set up SCons
+ shell: bash
+ run: |
+ python -c "import sys; print(sys.version)"
+ python -m pip install scons
+ scons --version
+
+ - name: Linux dependencies
+ if: ${{ matrix.platform == 'linux' }}
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -qqq build-essential pkg-config
+ g++ --version
+ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12
+ sudo update-alternatives --set g++ /usr/bin/g++-12
+ g++ --version
+
+ - name: Setup MinGW for Windows/MinGW build
+ if: ${{ matrix.platform == 'windows' }}
+ uses: OpenVicProject/mingw-cache@master
+
+ - name: Compile with SCons
+ uses: OpenVicProject/openvic-build@master
+ with:
+ platform: ${{ matrix.platform }}
+ target: ${{ matrix.target }}
+ sconsflags: arch=${{ matrix.arch }} build_ovsim_library=yes
+
+ - name: Delete compilation files
+ if: ${{ matrix.platform == 'windows' }}
+ run: |
+ Remove-Item bin/* -Include *.exp,*.pdb -Force
+
+ - name: Upload library artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ github.event.repository.name }}-library
+ path: |
+ ${{ github.workspace }}/bin/libopenvic-simulation.*
+
+ - name: Upload executable artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ github.event.repository.name }}-executable
+ path: |
+ ${{ github.workspace }}/bin/openvic-simulation.headless.*
+
+ - name: Archive Release
+ uses: thedoctor0/zip-release@0.7.1
+ with:
+ type: "zip"
+ filename: "../../../libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip"
+ directory: "${{ github.workspace }}/bin/"
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+
+ - name: Create and upload asset
+ uses: ncipollo/release-action@v1
+ with:
+ allowUpdates: true
+ artifacts: "libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip"
+ omitNameDuringUpdate: true
+ omitBodyDuringUpdate: true
+ token: ${{ secrets.GITHUB_TOKEN }}
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
diff --git a/.gitmodules b/.gitmodules
index df89ab5..1728f7a 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,6 @@
[submodule "deps/openvic-dataloader"]
path = deps/openvic-dataloader
url = https://github.com/OpenVicProject/OpenVic-Dataloader
+[submodule "scripts"]
+ path = scripts
+ url = git@github.com:OpenVicProject/scripts
diff --git a/SConstruct b/SConstruct
index 6fa567a..9ff3eec 100644
--- a/SConstruct
+++ b/SConstruct
@@ -1,219 +1,25 @@
#!/usr/bin/env python
-# This file is heavily based on https://github.com/godotengine/godot-cpp/blob/8155f35b29b4b08bc54b2eb0c57e1e9effe9f093/SConstruct
import os
import platform
import sys
-import importlib
import SCons
-# Local
-from scripts.build.option_handler import OptionsClass
-from scripts.build.glob_recursive import GlobRecursive
-from scripts.build.cache import show_progress
-
-# Try to detect the host platform automatically.
-# This is used if no `platform` argument is passed
-if sys.platform.startswith("linux"):
- default_platform = "linux"
-elif sys.platform == "darwin":
- default_platform = "macos"
-elif sys.platform == "win32" or sys.platform == "msys":
- default_platform = "windows"
-elif ARGUMENTS.get("platform", ""):
- default_platform = ARGUMENTS.get("platform")
-else:
- raise ValueError("Could not detect platform automatically, please specify with platform=<platform>")
-
-is_standalone = SCons.Script.sconscript_reading == 1
-
BINDIR = "bin"
-TOOLPATH = ["deps/openvic-dataloader/tools"]
-try:
- Import("env")
- old_env = env
- env = old_env.Clone()
-except:
- # Default tools with no platform defaults to gnu toolchain.
- # We apply platform specific toolchains via our custom tools.
- env = Environment(tools=["default"], PLATFORM="")
- old_env = env
+env = SConscript("scripts/SConstruct")
env.PrependENVPath("PATH", os.getenv("PATH"))
-# Default num_jobs to local cpu count if not user specified.
-# SCons has a peculiarity where user-specified options won't be overridden
-# by SetOption, so we can rely on this to know if we should use our default.
-initial_num_jobs = env.GetOption("num_jobs")
-altered_num_jobs = initial_num_jobs + 1
-env.SetOption("num_jobs", altered_num_jobs)
-if env.GetOption("num_jobs") == altered_num_jobs:
- cpu_count = os.cpu_count()
- if cpu_count is None:
- print("Couldn't auto-detect CPU count to configure build parallelism. Specify it with the -j argument.")
- else:
- safer_cpu_count = cpu_count if cpu_count <= 4 else cpu_count - 1
- print(
- "Auto-detected %d CPU cores available for build parallelism. Using %d cores by default. You can override it with the -j argument."
- % (cpu_count, safer_cpu_count)
- )
- env.SetOption("num_jobs", safer_cpu_count)
-
-opts = OptionsClass(ARGUMENTS)
-
-platforms = ("linux", "macos", "windows", "android", "ios", "javascript")
-unsupported_known_platforms = ("android", "ios", "javascript")
-opts.Add(
- EnumVariable(
- key="platform",
- help="Target platform",
- default=env.get("platform", default_platform),
- allowed_values=platforms,
- ignorecase=2,
- )
-)
-
-opts.Add(
- EnumVariable(
- key="target",
- help="Compilation target",
- default=env.get("target", "template_debug"),
- allowed_values=("editor", "template_release", "template_debug"),
- )
-)
-
-opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not is_standalone)))
-opts.Add(
- EnumVariable(
- key="precision",
- help="Set the floating-point precision level",
- default=env.get("precision", "single"),
- allowed_values=("single", "double"),
- )
-)
-
-# Add platform options
-tools = {}
-for pl in set(platforms) - set(unsupported_known_platforms):
- tool = Tool(pl, toolpath=TOOLPATH)
- if hasattr(tool, "options"):
- tool.options(opts)
- tools[pl] = tool
-
-# CPU architecture options.
-architecture_array = ["", "universal", "x86_32", "x86_64", "arm32", "arm64", "rv64", "ppc32", "ppc64", "wasm32"]
-architecture_aliases = {
- "x64": "x86_64",
- "amd64": "x86_64",
- "armv7": "arm32",
- "armv8": "arm64",
- "arm64v8": "arm64",
- "aarch64": "arm64",
- "rv": "rv64",
- "riscv": "rv64",
- "riscv64": "rv64",
- "ppcle": "ppc32",
- "ppc": "ppc32",
- "ppc64le": "ppc64",
-}
-opts.Add(
- EnumVariable(
- key="arch",
- help="CPU architecture",
- default=env.get("arch", ""),
- allowed_values=architecture_array,
- map=architecture_aliases,
- )
-)
-
-opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", is_standalone))
-
-opts.Add(BoolVariable("compiledb", "Generate compilation DB (`compile_commands.json`) for external tools", False))
-opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
-opts.Add(BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True))
-opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True))
-
-# Targets flags tool (optimizations, debug symbols)
-target_tool = Tool("targets", toolpath=TOOLPATH)
-target_tool.options(opts)
-
-# Custom options and profile flags.
-opts.Make(["custom.py"])
-opts.Finalize(env)
-Help(opts.GenerateHelpText(env))
-
-if env["platform"] in unsupported_known_platforms:
- print("Unsupported platform: " + env["platform"]+". Only supports " + ", ".join(set(platforms) - set(unsupported_known_platforms)))
- Exit()
-
-# Process CPU architecture argument.
-if env["arch"] == "":
- # No architecture specified. Default to arm64 if building for Android,
- # universal if building for macOS or iOS, wasm32 if building for web,
- # otherwise default to the host architecture.
- if env["platform"] in ["macos", "ios"]:
- env["arch"] = "universal"
- elif env["platform"] == "android":
- env["arch"] = "arm64"
- elif env["platform"] == "javascript":
- env["arch"] = "wasm32"
- else:
- host_machine = platform.machine().lower()
- if host_machine in architecture_array:
- env["arch"] = host_machine
- elif host_machine in architecture_aliases.keys():
- env["arch"] = architecture_aliases[host_machine]
- elif "86" in host_machine:
- # Catches x86, i386, i486, i586, i686, etc.
- env["arch"] = "x86_32"
- else:
- print("Unsupported CPU architecture: " + host_machine)
- Exit()
-
-tool = Tool(env["platform"], toolpath=TOOLPATH)
-
-if tool is None or not tool.exists(env):
- raise ValueError("Required toolchain not found for platform " + env["platform"])
-
-tool.generate(env)
-target_tool.generate(env)
-
-print("Building for architecture " + env["arch"] + " on platform " + env["platform"])
-
-# Require C++20
-if env.get("is_msvc", False):
- env.Append(CXXFLAGS=["/std:c++20"])
-else:
- env.Append(CXXFLAGS=["-std=c++20"])
-
-if env["precision"] == "double":
- env.Append(CPPDEFINES=["REAL_T_IS_DOUBLE"])
-
-scons_cache_path = os.environ.get("SCONS_CACHE")
-if scons_cache_path != None:
- CacheDir(scons_cache_path)
- print("Scons cache enabled... (path: '" + scons_cache_path + "')")
-
-if env["compiledb"]:
- # Generating the compilation DB (`compile_commands.json`) requires SCons 4.0.0 or later.
- from SCons import __version__ as scons_raw_version
-
- scons_ver = env._get_major_minor_revision(scons_raw_version)
-
- if scons_ver < (4, 0, 0):
- print("The `compiledb=yes` option requires SCons 4.0 or later, but your version is %s." % scons_raw_version)
- Exit(255)
+opts = env.SetupOptions()
- env.Tool("compilation_db")
- env.Alias("compiledb", env.CompilationDatabase())
+opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not env.is_standalone)))
+opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", env.is_standalone))
-ovdl_env = SConscript("deps/openvic-dataloader/SConstruct")
+env.FinalizeOptions()
-env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"])
-env.Append(LIBS=ovdl_env.openvic_dataloader["LIBS"])
-env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"])
+SConscript("deps/SCsub")
env.openvic_simulation = {}
@@ -229,7 +35,7 @@ env.openvic_simulation = {}
source_path = "src/openvic-simulation"
include_path = "src"
env.Append(CPPPATH=[[env.Dir(p) for p in [source_path, include_path]]])
-sources = GlobRecursive("*.cpp", [source_path])
+sources = env.GlobRecursive("*.cpp", [source_path])
env.simulation_sources = sources
suffix = ".{}.{}".format(env["platform"], env["target"])
@@ -251,11 +57,11 @@ if env["build_ovsim_library"]:
Default(library)
env.Append(LIBPATH=[env.Dir(BINDIR)])
- env.Append(LIBS=[library_name])
+ env.Prepend(LIBS=[library_name])
env.openvic_simulation["LIBPATH"] = env["LIBPATH"]
env.openvic_simulation["LIBS"] = env["LIBS"]
- env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + ovdl_env.openvic_dataloader["INCPATH"]
+ env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + env.openvic_dataloader["INCPATH"]
headless_program = None
env["PROGSUFFIX"] = suffix + env["PROGSUFFIX"]
@@ -266,7 +72,7 @@ if env["build_ovsim_headless"]:
headless_path = ["src/headless"]
headless_env.Append(CPPDEFINES=["OPENVIC_SIM_HEADLESS"])
headless_env.Append(CPPPATH=[headless_env.Dir(headless_path)])
- headless_env.headless_sources = GlobRecursive("*.cpp", headless_path)
+ headless_env.headless_sources = env.GlobRecursive("*.cpp", headless_path)
if not env["build_ovsim_library"]:
headless_env.headless_sources += sources
headless_program = headless_env.Program(
@@ -279,6 +85,6 @@ if env["build_ovsim_headless"]:
if "env" in locals():
# FIXME: This method mixes both cosmetic progress stuff and cache handling...
- show_progress(env)
+ env.show_progress(env)
Return("env")
diff --git a/deps/SCsub b/deps/SCsub
new file mode 100644
index 0000000..7c945e6
--- /dev/null
+++ b/deps/SCsub
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+Import("env")
+
+def build_openvic_dataloader(env):
+ ovdl_env = SConscript("openvic-dataloader/SConstruct")
+ env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"])
+ env.Prepend(LIBS=ovdl_env.openvic_dataloader["LIBS"])
+ env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"])
+ env.openvic_dataloader = ovdl_env.openvic_dataloader
+
+build_openvic_dataloader(env) \ No newline at end of file
diff --git a/deps/openvic-dataloader b/deps/openvic-dataloader
-Subproject 977661f6f4301be19fa64abfc6cda5040c3899b
+Subproject 706c2413c6b390a18389414493218c21a59f495
diff --git a/scripts b/scripts
new file mode 160000
+Subproject 71803e491af7174f536e72f76dfb74708b4b84f
diff --git a/scripts/build/cache.py b/scripts/build/cache.py
deleted file mode 100644
index d48b0e0..0000000
--- a/scripts/build/cache.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172
-def show_progress(env):
- import os
- import sys
- import glob
- from SCons.Script import Progress, Command, AlwaysBuild
-
- screen = sys.stdout
- # Progress reporting is not available in non-TTY environments since it
- # messes with the output (for example, when writing to a file)
- show_progress = env["progress"] and sys.stdout.isatty()
- node_count = 0
- node_count_max = 0
- node_count_interval = 1
- node_count_fname = str(env.Dir("#")) + "/.scons_node_count"
-
- import time, math
-
- class cache_progress:
- # The default is 1 GB cache and 12 hours half life
- def __init__(self, path=None, limit=1073741824, half_life=43200):
- self.path = path
- self.limit = limit
- self.exponent_scale = math.log(2) / half_life
- if env["verbose"] and path != None:
- screen.write(
- "Current cache limit is {} (used: {})\n".format(
- self.convert_size(limit), self.convert_size(self.get_size(path))
- )
- )
- self.delete(self.file_list())
-
- def __call__(self, node, *args, **kw):
- nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress
- if show_progress:
- # Print the progress percentage
- node_count += node_count_interval
- if node_count_max > 0 and node_count <= node_count_max:
- screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max))
- screen.flush()
- elif node_count_max > 0 and node_count > node_count_max:
- screen.write("\r[100%] ")
- screen.flush()
- else:
- screen.write("\r[Initial build] ")
- screen.flush()
-
- def delete(self, files):
- if len(files) == 0:
- return
- if env["verbose"]:
- # Utter something
- screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file"))
- [os.remove(f) for f in files]
-
- def file_list(self):
- if self.path is None:
- # Nothing to do
- return []
- # Gather a list of (filename, (size, atime)) within the
- # cache directory
- file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))]
- if file_stat == []:
- # Nothing to do
- return []
- # Weight the cache files by size (assumed to be roughly
- # proportional to the recompilation time) times an exponential
- # decay since the ctime, and return a list with the entries
- # (filename, size, weight).
- current_time = time.time()
- file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat]
- # Sort by the most recently accessed files (most sensible to keep) first
- file_stat.sort(key=lambda x: x[2])
- # Search for the first entry where the storage limit is
- # reached
- sum, mark = 0, None
- for i, x in enumerate(file_stat):
- sum += x[1]
- if sum > self.limit:
- mark = i
- break
- if mark is None:
- return []
- else:
- return [x[0] for x in file_stat[mark:]]
-
- def convert_size(self, size_bytes):
- if size_bytes == 0:
- return "0 bytes"
- size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
- i = int(math.floor(math.log(size_bytes, 1024)))
- p = math.pow(1024, i)
- s = round(size_bytes / p, 2)
- return "%s %s" % (int(s) if i == 0 else s, size_name[i])
-
- def get_size(self, start_path="."):
- total_size = 0
- for dirpath, dirnames, filenames in os.walk(start_path):
- for f in filenames:
- fp = os.path.join(dirpath, f)
- total_size += os.path.getsize(fp)
- return total_size
-
- def progress_finish(target, source, env):
- nonlocal node_count, progressor
- try:
- with open(node_count_fname, "w") as f:
- f.write("%d\n" % node_count)
- progressor.delete(progressor.file_list())
- except Exception:
- pass
-
- try:
- with open(node_count_fname) as f:
- node_count_max = int(f.readline())
- except Exception:
- pass
-
- cache_directory = os.environ.get("SCONS_CACHE")
- # Simple cache pruning, attached to SCons' progress callback. Trim the
- # cache directory to a size not larger than cache_limit.
- cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024
- progressor = cache_progress(cache_directory, cache_limit)
- Progress(progressor, interval=node_count_interval)
-
- progress_finish_command = Command("progress_finish", [], progress_finish)
- AlwaysBuild(progress_finish_command)
diff --git a/scripts/build/glob_recursive.py b/scripts/build/glob_recursive.py
deleted file mode 100644
index db6eb80..0000000
--- a/scripts/build/glob_recursive.py
+++ /dev/null
@@ -1,15 +0,0 @@
-def GlobRecursive(pattern, nodes=['.']):
- import SCons
- import glob
- fs = SCons.Node.FS.get_default_fs()
- Glob = fs.Glob
-
- results = []
- for node in nodes:
- nnodes = []
- for f in Glob(str(node) + '/*', source=True):
- if type(f) is SCons.Node.FS.Dir:
- nnodes.append(f)
- results += GlobRecursive(pattern, nnodes)
- results += Glob(str(node) + '/' + pattern, source=True)
- return results
diff --git a/scripts/build/option_handler.py b/scripts/build/option_handler.py
deleted file mode 100644
index 3cebc1a..0000000
--- a/scripts/build/option_handler.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from typing import Tuple, Iterable
-from SCons.Variables import Variables
-
-class OptionsClass:
- def __init__(self, args):
- self.opts = None
- self.opt_list = []
- self.args = args
- self.saved_args = args.copy()
-
- def Add(self, variableOrKey, *argv, **kwarg):
-
- self.opt_list.append([variableOrKey, argv, kwarg])
- # Neccessary to have our own build options without errors
- if isinstance(variableOrKey, str):
- self.args.pop(variableOrKey, True)
- else:
- self.args.pop(variableOrKey[0], True)
-
- def Make(self, customs : Iterable[str]):
- self.args = self.saved_args
- profile = self.args.get("profile", "")
- if profile:
- if os.path.isfile(profile):
- customs.append(profile)
- elif os.path.isfile(profile + ".py"):
- customs.append(profile + ".py")
- self.opts = Variables(customs, self.args)
- for opt in self.opt_list:
- if opt[1] == None and opt[2] == None:
- self.opts.Add(opt[0])
- else:
- self.opts.Add(opt[0], *opt[1], **opt[2])
-
- def Finalize(self, env):
- self.opts.Update(env)
-
- def GenerateHelpText(self, env):
- return self.opts.GenerateHelpText(env)