Merge pull request #26 from OpenVicProject/outsource/extras

9 files changed, 167 insertions, 386 deletions

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
new file mode 100644
index 0000000..71c4054
--- /dev/null
+++ b/.github/workflows/builds.yml
@@ -0,0 +1,141 @@
+name: Builds
+
+on: [push, pull_request]
+
+env:
+  GH_BASE_BRANCH: master
+
+concurrency:
+  group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-macos
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ${{matrix.os}}
+    name: ${{matrix.name}}
+    permissions: write-all
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - identifier: windows-debug
+            os: windows-latest
+            name: 🏁 Windows Debug
+            target: template_debug
+            platform: windows
+            arch: x86_64
+          - identifier: windows-release
+            os: windows-latest
+            name: 🏁 Windows Release
+            target: template_release
+            platform: windows
+            arch: x86_64
+          - identifier: macos-debug
+            os: macos-latest
+            name: 🍎 macOS (universal) Debug
+            target: template_debug
+            platform: macos
+            arch: universal
+          - identifier: macos-release
+            os: macos-latest
+            name: 🍎 macOS (universal) Release
+            target: template_release
+            platform: macos
+            arch: universal
+          - identifier: linux-debug
+            os: ubuntu-latest
+            name: 🐧 Linux Debug
+            runner: ubuntu-20.04
+            target: template_debug
+            platform: linux
+            arch: x86_64
+          - identifier: linux-release
+            os: ubuntu-latest
+            name: 🐧 Linux Release
+            runner: ubuntu-20.04
+            target: template_release
+            platform: linux
+            arch: x86_64
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3.3.0
+        with:
+          submodules: recursive
+
+      - name: Setup build cache
+        uses: OpenVicProject/openvic-cache@master
+        with:
+          cache-name: ${{ matrix.identifier }}
+          base-branch: ${{ env.GH_BASE_BRANCH }}
+        continue-on-error: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - name: Set up SCons
+        shell: bash
+        run: |
+          python -c "import sys; print(sys.version)"
+          python -m pip install scons
+          scons --version
+
+      - name: Linux dependencies
+        if: ${{ matrix.platform == 'linux' }}
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qqq build-essential pkg-config
+          g++ --version
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12
+          sudo update-alternatives --set g++ /usr/bin/g++-12
+          g++ --version
+
+      - name: Setup MinGW for Windows/MinGW build
+        if: ${{ matrix.platform == 'windows' }}
+        uses: OpenVicProject/mingw-cache@master
+
+      - name: Compile with SCons
+        uses: OpenVicProject/openvic-build@master
+        with:
+          platform: ${{ matrix.platform }}
+          target: ${{ matrix.target }}
+          sconsflags: arch=${{ matrix.arch }} build_ovsim_library=yes
+
+      - name: Delete compilation files
+        if: ${{ matrix.platform == 'windows' }}
+        run: |
+          Remove-Item bin/* -Include *.exp,*.pdb -Force
+
+      - name: Upload library artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.event.repository.name }}-library
+          path: |
+            ${{ github.workspace }}/bin/libopenvic-simulation.*
+
+      - name: Upload executable artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.event.repository.name }}-executable
+          path: |
+            ${{ github.workspace }}/bin/openvic-simulation.headless.*
+
+      - name: Archive Release
+        uses: thedoctor0/zip-release@0.7.1
+        with:
+          type: "zip"
+          filename: "../../../libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip"
+          directory: "${{ github.workspace }}/bin/"
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+
+      - name: Create and upload asset
+        uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: true
+          artifacts: "libopenvic-simulation.${{ matrix.platform }}.${{ matrix.arch }}.zip"
+          omitNameDuringUpdate: true
+          omitBodyDuringUpdate: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
diff --git a/.gitmodules b/.gitmodules
index df89ab5..1728f7a 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "deps/openvic-dataloader"]
 	path = deps/openvic-dataloader
 	url = https://github.com/OpenVicProject/OpenVic-Dataloader
+[submodule "scripts"]
+	path = scripts
+	url = git@github.com:OpenVicProject/scripts
diff --git a/SConstruct b/SConstruct
index 6fa567a..9ff3eec 100644
--- a/SConstruct
+++ b/SConstruct
@@ -1,219 +1,25 @@
 #!/usr/bin/env python
 
-# This file is heavily based on https://github.com/godotengine/godot-cpp/blob/8155f35b29b4b08bc54b2eb0c57e1e9effe9f093/SConstruct
 import os
 import platform
 import sys
-import importlib
 
 import SCons
 
-# Local
-from scripts.build.option_handler import OptionsClass
-from scripts.build.glob_recursive import GlobRecursive
-from scripts.build.cache import show_progress
-
-# Try to detect the host platform automatically.
-# This is used if no `platform` argument is passed
-if sys.platform.startswith("linux"):
-    default_platform = "linux"
-elif sys.platform == "darwin":
-    default_platform = "macos"
-elif sys.platform == "win32" or sys.platform == "msys":
-    default_platform = "windows"
-elif ARGUMENTS.get("platform", ""):
-    default_platform = ARGUMENTS.get("platform")
-else:
-    raise ValueError("Could not detect platform automatically, please specify with platform=<platform>")
-
-is_standalone = SCons.Script.sconscript_reading == 1
-
 BINDIR = "bin"
-TOOLPATH = ["deps/openvic-dataloader/tools"]
 
-try:
-    Import("env")
-    old_env = env
-    env = old_env.Clone()
-except:
-    # Default tools with no platform defaults to gnu toolchain.
-    # We apply platform specific toolchains via our custom tools.
-    env = Environment(tools=["default"], PLATFORM="")
-    old_env = env
+env = SConscript("scripts/SConstruct")
 
 env.PrependENVPath("PATH", os.getenv("PATH"))
 
-# Default num_jobs to local cpu count if not user specified.
-# SCons has a peculiarity where user-specified options won't be overridden
-# by SetOption, so we can rely on this to know if we should use our default.
-initial_num_jobs = env.GetOption("num_jobs")
-altered_num_jobs = initial_num_jobs + 1
-env.SetOption("num_jobs", altered_num_jobs)
-if env.GetOption("num_jobs") == altered_num_jobs:
-    cpu_count = os.cpu_count()
-    if cpu_count is None:
-        print("Couldn't auto-detect CPU count to configure build parallelism. Specify it with the -j argument.")
-    else:
-        safer_cpu_count = cpu_count if cpu_count <= 4 else cpu_count - 1
-        print(
-            "Auto-detected %d CPU cores available for build parallelism. Using %d cores by default. You can override it with the -j argument."
-            % (cpu_count, safer_cpu_count)
-        )
-        env.SetOption("num_jobs", safer_cpu_count)
-
-opts = OptionsClass(ARGUMENTS)
-
-platforms = ("linux", "macos", "windows", "android", "ios", "javascript")
-unsupported_known_platforms = ("android", "ios", "javascript")
-opts.Add(
-    EnumVariable(
-        key="platform",
-        help="Target platform",
-        default=env.get("platform", default_platform),
-        allowed_values=platforms,
-        ignorecase=2,
-    )
-)
-
-opts.Add(
-    EnumVariable(
-        key="target",
-        help="Compilation target",
-        default=env.get("target", "template_debug"),
-        allowed_values=("editor", "template_release", "template_debug"),
-    )
-)
-
-opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not is_standalone)))
-opts.Add(
-    EnumVariable(
-        key="precision",
-        help="Set the floating-point precision level",
-        default=env.get("precision", "single"),
-        allowed_values=("single", "double"),
-    )
-)
-
-# Add platform options
-tools = {}
-for pl in set(platforms) - set(unsupported_known_platforms):
-    tool = Tool(pl, toolpath=TOOLPATH)
-    if hasattr(tool, "options"):
-        tool.options(opts)
-    tools[pl] = tool
-
-# CPU architecture options.
-architecture_array = ["", "universal", "x86_32", "x86_64", "arm32", "arm64", "rv64", "ppc32", "ppc64", "wasm32"]
-architecture_aliases = {
-    "x64": "x86_64",
-    "amd64": "x86_64",
-    "armv7": "arm32",
-    "armv8": "arm64",
-    "arm64v8": "arm64",
-    "aarch64": "arm64",
-    "rv": "rv64",
-    "riscv": "rv64",
-    "riscv64": "rv64",
-    "ppcle": "ppc32",
-    "ppc": "ppc32",
-    "ppc64le": "ppc64",
-}
-opts.Add(
-    EnumVariable(
-        key="arch",
-        help="CPU architecture",
-        default=env.get("arch", ""),
-        allowed_values=architecture_array,
-        map=architecture_aliases,
-    )
-)
-
-opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", is_standalone))
-
-opts.Add(BoolVariable("compiledb", "Generate compilation DB (`compile_commands.json`) for external tools", False))
-opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
-opts.Add(BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True))
-opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True))
-
-# Targets flags tool (optimizations, debug symbols)
-target_tool = Tool("targets", toolpath=TOOLPATH)
-target_tool.options(opts)
-
-# Custom options and profile flags.
-opts.Make(["custom.py"])
-opts.Finalize(env)
-Help(opts.GenerateHelpText(env))
-
-if env["platform"] in unsupported_known_platforms:
-    print("Unsupported platform: " + env["platform"]+". Only supports " + ", ".join(set(platforms) - set(unsupported_known_platforms)))
-    Exit()
-
-# Process CPU architecture argument.
-if env["arch"] == "":
-    # No architecture specified. Default to arm64 if building for Android,
-    # universal if building for macOS or iOS, wasm32 if building for web,
-    # otherwise default to the host architecture.
-    if env["platform"] in ["macos", "ios"]:
-        env["arch"] = "universal"
-    elif env["platform"] == "android":
-        env["arch"] = "arm64"
-    elif env["platform"] == "javascript":
-        env["arch"] = "wasm32"
-    else:
-        host_machine = platform.machine().lower()
-        if host_machine in architecture_array:
-            env["arch"] = host_machine
-        elif host_machine in architecture_aliases.keys():
-            env["arch"] = architecture_aliases[host_machine]
-        elif "86" in host_machine:
-            # Catches x86, i386, i486, i586, i686, etc.
-            env["arch"] = "x86_32"
-        else:
-            print("Unsupported CPU architecture: " + host_machine)
-            Exit()
-
-tool = Tool(env["platform"], toolpath=TOOLPATH)
-
-if tool is None or not tool.exists(env):
-    raise ValueError("Required toolchain not found for platform " + env["platform"])
-
-tool.generate(env)
-target_tool.generate(env)
-
-print("Building for architecture " + env["arch"] + " on platform " + env["platform"])
-
-# Require C++20
-if env.get("is_msvc", False):
-    env.Append(CXXFLAGS=["/std:c++20"])
-else:
-    env.Append(CXXFLAGS=["-std=c++20"])
-
-if env["precision"] == "double":
-    env.Append(CPPDEFINES=["REAL_T_IS_DOUBLE"])
-
-scons_cache_path = os.environ.get("SCONS_CACHE")
-if scons_cache_path != None:
-    CacheDir(scons_cache_path)
-    print("Scons cache enabled... (path: '" + scons_cache_path + "')")
-
-if env["compiledb"]:
-    # Generating the compilation DB (`compile_commands.json`) requires SCons 4.0.0 or later.
-    from SCons import __version__ as scons_raw_version
-
-    scons_ver = env._get_major_minor_revision(scons_raw_version)
-
-    if scons_ver < (4, 0, 0):
-        print("The `compiledb=yes` option requires SCons 4.0 or later, but your version is %s." % scons_raw_version)
-        Exit(255)
+opts = env.SetupOptions()
 
-    env.Tool("compilation_db")
-    env.Alias("compiledb", env.CompilationDatabase())
+opts.Add(BoolVariable(key="build_ovsim_library", help="Build the openvic simulation library.", default=env.get("build_ovsim_library", not env.is_standalone)))
+opts.Add(BoolVariable("build_ovsim_headless", "Build the openvic simulation headless executable", env.is_standalone))
 
-ovdl_env = SConscript("deps/openvic-dataloader/SConstruct")
+env.FinalizeOptions()
 
-env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"])
-env.Append(LIBS=ovdl_env.openvic_dataloader["LIBS"])
-env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"])
+SConscript("deps/SCsub")
 
 env.openvic_simulation = {}
 
@@ -229,7 +35,7 @@ env.openvic_simulation = {}
 source_path = "src/openvic-simulation"
 include_path = "src"
 env.Append(CPPPATH=[[env.Dir(p) for p in [source_path, include_path]]])
-sources = GlobRecursive("*.cpp", [source_path])
+sources = env.GlobRecursive("*.cpp", [source_path])
 env.simulation_sources = sources
 
 suffix = ".{}.{}".format(env["platform"], env["target"])
@@ -251,11 +57,11 @@ if env["build_ovsim_library"]:
     Default(library)
 
     env.Append(LIBPATH=[env.Dir(BINDIR)])
-    env.Append(LIBS=[library_name])
+    env.Prepend(LIBS=[library_name])
 
     env.openvic_simulation["LIBPATH"] = env["LIBPATH"]
     env.openvic_simulation["LIBS"] = env["LIBS"]
-    env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + ovdl_env.openvic_dataloader["INCPATH"]
+    env.openvic_simulation["INCPATH"] = [env.Dir(include_path)] + env.openvic_dataloader["INCPATH"]
 
 headless_program = None
 env["PROGSUFFIX"] = suffix + env["PROGSUFFIX"]
@@ -266,7 +72,7 @@ if env["build_ovsim_headless"]:
     headless_path = ["src/headless"]
     headless_env.Append(CPPDEFINES=["OPENVIC_SIM_HEADLESS"])
     headless_env.Append(CPPPATH=[headless_env.Dir(headless_path)])
-    headless_env.headless_sources = GlobRecursive("*.cpp", headless_path)
+    headless_env.headless_sources = env.GlobRecursive("*.cpp", headless_path)
     if not env["build_ovsim_library"]:
         headless_env.headless_sources += sources
     headless_program = headless_env.Program(
@@ -279,6 +85,6 @@ if env["build_ovsim_headless"]:
 
 if "env" in locals():
     # FIXME: This method mixes both cosmetic progress stuff and cache handling...
-    show_progress(env)
+    env.show_progress(env)
 
 Return("env")
diff --git a/deps/SCsub b/deps/SCsub
new file mode 100644
index 0000000..7c945e6
--- /dev/null
+++ b/deps/SCsub
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+Import("env")
+
+def build_openvic_dataloader(env):
+    ovdl_env = SConscript("openvic-dataloader/SConstruct")
+    env.Append(LIBPATH=ovdl_env.openvic_dataloader["LIBPATH"])
+    env.Prepend(LIBS=ovdl_env.openvic_dataloader["LIBS"])
+    env.Append(CPPPATH=ovdl_env.openvic_dataloader["INCPATH"])
+    env.openvic_dataloader = ovdl_env.openvic_dataloader
+
+build_openvic_dataloader(env)
\ No newline at end of file
diff --git a/deps/openvic-dataloader b/deps/openvic-dataloader
-Subproject 977661f6f4301be19fa64abfc6cda5040c3899b
+Subproject 706c2413c6b390a18389414493218c21a59f495
diff --git a/scripts b/scripts
new file mode 160000
+Subproject 71803e491af7174f536e72f76dfb74708b4b84f
diff --git a/scripts/build/cache.py b/scripts/build/cache.py
deleted file mode 100644
index d48b0e0..0000000
--- a/scripts/build/cache.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172
-def show_progress(env):
-    import os
-    import sys
-    import glob
-    from SCons.Script import Progress, Command, AlwaysBuild
-
-    screen = sys.stdout
-    # Progress reporting is not available in non-TTY environments since it
-    # messes with the output (for example, when writing to a file)
-    show_progress = env["progress"] and sys.stdout.isatty()
-    node_count = 0
-    node_count_max = 0
-    node_count_interval = 1
-    node_count_fname = str(env.Dir("#")) + "/.scons_node_count"
-
-    import time, math
-
-    class cache_progress:
-        # The default is 1 GB cache and 12 hours half life
-        def __init__(self, path=None, limit=1073741824, half_life=43200):
-            self.path = path
-            self.limit = limit
-            self.exponent_scale = math.log(2) / half_life
-            if env["verbose"] and path != None:
-                screen.write(
-                    "Current cache limit is {} (used: {})\n".format(
-                        self.convert_size(limit), self.convert_size(self.get_size(path))
-                    )
-                )
-            self.delete(self.file_list())
-
-        def __call__(self, node, *args, **kw):
-            nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress
-            if show_progress:
-                # Print the progress percentage
-                node_count += node_count_interval
-                if node_count_max > 0 and node_count <= node_count_max:
-                    screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max))
-                    screen.flush()
-                elif node_count_max > 0 and node_count > node_count_max:
-                    screen.write("\r[100%] ")
-                    screen.flush()
-                else:
-                    screen.write("\r[Initial build] ")
-                    screen.flush()
-
-        def delete(self, files):
-            if len(files) == 0:
-                return
-            if env["verbose"]:
-                # Utter something
-                screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file"))
-            [os.remove(f) for f in files]
-
-        def file_list(self):
-            if self.path is None:
-                # Nothing to do
-                return []
-            # Gather a list of (filename, (size, atime)) within the
-            # cache directory
-            file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))]
-            if file_stat == []:
-                # Nothing to do
-                return []
-            # Weight the cache files by size (assumed to be roughly
-            # proportional to the recompilation time) times an exponential
-            # decay since the ctime, and return a list with the entries
-            # (filename, size, weight).
-            current_time = time.time()
-            file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat]
-            # Sort by the most recently accessed files (most sensible to keep) first
-            file_stat.sort(key=lambda x: x[2])
-            # Search for the first entry where the storage limit is
-            # reached
-            sum, mark = 0, None
-            for i, x in enumerate(file_stat):
-                sum += x[1]
-                if sum > self.limit:
-                    mark = i
-                    break
-            if mark is None:
-                return []
-            else:
-                return [x[0] for x in file_stat[mark:]]
-
-        def convert_size(self, size_bytes):
-            if size_bytes == 0:
-                return "0 bytes"
-            size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
-            i = int(math.floor(math.log(size_bytes, 1024)))
-            p = math.pow(1024, i)
-            s = round(size_bytes / p, 2)
-            return "%s %s" % (int(s) if i == 0 else s, size_name[i])
-
-        def get_size(self, start_path="."):
-            total_size = 0
-            for dirpath, dirnames, filenames in os.walk(start_path):
-                for f in filenames:
-                    fp = os.path.join(dirpath, f)
-                    total_size += os.path.getsize(fp)
-            return total_size
-
-    def progress_finish(target, source, env):
-        nonlocal node_count, progressor
-        try:
-            with open(node_count_fname, "w") as f:
-                f.write("%d\n" % node_count)
-            progressor.delete(progressor.file_list())
-        except Exception:
-            pass
-
-    try:
-        with open(node_count_fname) as f:
-            node_count_max = int(f.readline())
-    except Exception:
-        pass
-
-    cache_directory = os.environ.get("SCONS_CACHE")
-    # Simple cache pruning, attached to SCons' progress callback. Trim the
-    # cache directory to a size not larger than cache_limit.
-    cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024
-    progressor = cache_progress(cache_directory, cache_limit)
-    Progress(progressor, interval=node_count_interval)
-
-    progress_finish_command = Command("progress_finish", [], progress_finish)
-    AlwaysBuild(progress_finish_command)
diff --git a/scripts/build/glob_recursive.py b/scripts/build/glob_recursive.py
deleted file mode 100644
index db6eb80..0000000
--- a/scripts/build/glob_recursive.py
+++ /dev/null
@@ -1,15 +0,0 @@
-def GlobRecursive(pattern, nodes=['.']):
-    import SCons
-    import glob
-    fs = SCons.Node.FS.get_default_fs()
-    Glob = fs.Glob
-
-    results = []
-    for node in nodes:
-        nnodes = []
-        for f in Glob(str(node) + '/*', source=True):
-            if type(f) is SCons.Node.FS.Dir:
-                nnodes.append(f)
-        results += GlobRecursive(pattern, nnodes)
-        results += Glob(str(node) + '/' + pattern, source=True)
-    return results
diff --git a/scripts/build/option_handler.py b/scripts/build/option_handler.py
deleted file mode 100644
index 3cebc1a..0000000
--- a/scripts/build/option_handler.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from typing import Tuple, Iterable
-from SCons.Variables import Variables
-
-class OptionsClass:
-    def __init__(self, args):
-        self.opts = None
-        self.opt_list = []
-        self.args = args
-        self.saved_args = args.copy()
-
-    def Add(self, variableOrKey, *argv, **kwarg):
-
-        self.opt_list.append([variableOrKey, argv, kwarg])
-        # Neccessary to have our own build options without errors
-        if isinstance(variableOrKey, str):
-            self.args.pop(variableOrKey, True)
-        else:
-            self.args.pop(variableOrKey[0], True)
-
-    def Make(self, customs : Iterable[str]):
-        self.args = self.saved_args
-        profile = self.args.get("profile", "")
-        if profile:
-            if os.path.isfile(profile):
-                customs.append(profile)
-            elif os.path.isfile(profile + ".py"):
-                customs.append(profile + ".py")
-        self.opts = Variables(customs, self.args)
-        for opt in self.opt_list:
-            if opt[1] == None and opt[2] == None:
-                self.opts.Add(opt[0])
-            else:
-                self.opts.Add(opt[0], *opt[1], **opt[2])
-
-    def Finalize(self, env):
-        self.opts.Update(env)
-
-    def GenerateHelpText(self, env):
-        return self.opts.GenerateHelpText(env)