aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author Spartan322 <Megacake1234@gmail.com>2023-07-07 18:26:54 +0200
committer Spartan322 <Megacake1234@gmail.com>2023-07-09 01:18:04 +0200
commitae16937f45a4d339d579294b7b53e84beec4749e (patch)
tree50800449a7a936cd68e4ba437004929ed166df8f
parent5838c2508682bc3f6e35e44056f9ba229bca4571 (diff)
Add scons-cache support for extension build
Copies how Godot handles scons-cache Add openvic-build action Add openvic-cache action Add mingw-cache action Add verbose and progress Scons options to SConstruct This is completely based on how Godot implemented caching with Scons
-rw-r--r--.github/actions/mingw-cache/action.yml18
-rw-r--r--.github/actions/openvic-build/action.yml32
-rw-r--r--.github/actions/openvic-cache/action.yml22
-rw-r--r--.github/workflows/builds.yml17
-rw-r--r--SConstruct140
5 files changed, 225 insertions, 4 deletions
diff --git a/.github/actions/mingw-cache/action.yml b/.github/actions/mingw-cache/action.yml
new file mode 100644
index 0000000..e92962d
--- /dev/null
+++ b/.github/actions/mingw-cache/action.yml
@@ -0,0 +1,18 @@
+name: Setup MinGW Cache
+description: Setup MinGW Cache
+inputs:
+ cache-key:
+ description: Name for MinGW cache key.
+ default: "mingw-cache"
+runs:
+ using: "composite"
+ steps:
+ # Upload cache on completion and check it out now
+ - name: Load MinGW Cache
+ id: cache-mingw
+ uses: actions/cache@v3
+ with:
+ path: C:\ProgramData\chocolatey\lib\mingw
+ key: ${{inputs.cache-key}}
+ - name: Setup MinGW for build
+ uses: egor-tensin/setup-mingw@v2 \ No newline at end of file
diff --git a/.github/actions/openvic-build/action.yml b/.github/actions/openvic-build/action.yml
new file mode 100644
index 0000000..a4b3507
--- /dev/null
+++ b/.github/actions/openvic-build/action.yml
@@ -0,0 +1,32 @@
+name: Build OpenVic Extension
+description: Build OpenVic Extension with provided options.
+inputs:
+ target:
+ description: Build target (editor, template_release, template_debug).
+ default: "template_release"
+ platform:
+ description: Target platform.
+ required: false
+ sconsflags:
+ default: ""
+ scons-cache:
+ description: The scons cache path.
+ default: "${{ github.workspace }}/.scons-cache/"
+ scons-cache-limit:
+ description: The scons cache size limit.
+ # actions/cache has 10 GiB limit, and GitHub runners have a 14 GiB disk.
+ # Limit to 7 GiB to avoid having the extracted cache fill the disk.
+ default: 7168
+runs:
+ using: "composite"
+ steps:
+ - name: Scons Build
+ shell: sh
+ env:
+ SCONSFLAGS: ${{ inputs.sconsflags }}
+ SCONS_CACHE: ${{ inputs.scons-cache }}
+ SCONS_CACHE_LIMIT: ${{ inputs.scons-cache-limit }}
+ run: |
+ echo "Building with flags:" platform=${{ inputs.platform }} target=${{ inputs.target }} ${{ env.SCONSFLAGS }}
+ scons platform=${{ inputs.platform }} target=${{ inputs.target }} ${{ env.SCONSFLAGS }}
+ ls -l game/bin/openvic/ \ No newline at end of file
diff --git a/.github/actions/openvic-cache/action.yml b/.github/actions/openvic-cache/action.yml
new file mode 100644
index 0000000..b4164eb
--- /dev/null
+++ b/.github/actions/openvic-cache/action.yml
@@ -0,0 +1,22 @@
+name: Setup OpenVic Build Cache
+description: Setup OpenVic Build Cache
+inputs:
+ cache-name:
+ description: The cache base name (job name by default).
+ default: "${{github.job}}"
+ scons-cache:
+ description: The scons cache path.
+ default: "${{github.workspace}}/.scons-cache/"
+runs:
+ using: "composite"
+ steps:
+ # Upload cache on completion and check it out now
+ - name: Load .scons_cache directory
+ uses: actions/cache@v3
+ with:
+ path: ${{inputs.scons-cache}}
+ key: ${{inputs.cache-name}}-${{env.OPENVIC_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+ restore-keys: |
+ ${{inputs.cache-name}}-${{env.OPENVIC_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+ ${{inputs.cache-name}}-${{env.OPENVIC_BASE_BRANCH}}-${{github.ref}}
+ ${{inputs.cache-name}}-${{env.OPENVIC_BASE_BRANCH}} \ No newline at end of file
diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 79f04aa..9944a9c 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -7,6 +7,7 @@ env:
GODOT_VERSION_PREFIX: Godot_v
GODOT_VERSION_SUFFIX: stable
GODOT_VERSION: 4.1
+ OPENVIC_BASE_BRANCH: master
concurrency:
group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-macos
@@ -67,6 +68,12 @@ jobs:
with:
submodules: recursive
+ - name: Setup OpenVic build cache
+ uses: ./.github/actions/openvic-cache
+ with:
+ cache-name: ${{ matrix.identifier }}
+ continue-on-error: true
+
- name: Setup Environment
uses: ./.github/actions/openvic-env
@@ -94,12 +101,14 @@ jobs:
- name: Setup MinGW for Windows/MinGW build
if: ${{ matrix.platform == 'windows' }}
- uses: egor-tensin/setup-mingw@v2
+ uses: ./.github/actions/mingw-cache
- name: Compile Extension
- shell: sh
- run: |
- scons target='${{ matrix.target }}' platform='${{ matrix.platform }}' arch='${{ matrix.arch }}'
+ uses: ./.github/actions/openvic-build
+ with:
+ platform: ${{ matrix.platform }}
+ target: ${{ matrix.target }}
+ sconsflags: arch=${{ matrix.arch }}
- name: Delete compilation files
if: ${{ matrix.platform == 'windows' }}
diff --git a/SConstruct b/SConstruct
index 51a2a24..426682c 100644
--- a/SConstruct
+++ b/SConstruct
@@ -7,6 +7,8 @@ from pathlib import Path
# Neccessary to have our own build options without errors
SAVED_ARGUMENTS = ARGUMENTS.copy()
ARGUMENTS.pop('intermediate_delete', True)
+ARGUMENTS.pop('progress', True)
+ARGUMENTS.pop('verbose', True)
env = SConscript("godot-cpp/SConstruct")
@@ -28,9 +30,11 @@ if profile:
customs.append(profile + ".py")
opts = Variables(customs, ARGUMENTS)
+opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
opts.Add(
BoolVariable("intermediate_delete", "Enables automatically deleting unassociated intermediate binary files.", True)
)
+opts.Add(BoolVariable("progress", "Show a progress indicator during compilation", True))
opts.Update(env)
Help(opts.GenerateHelpText(env))
@@ -47,6 +51,138 @@ def GlobRecursive(pattern, nodes=['.']):
results += Glob(str(node) + '/' + pattern, source=True)
return results
+# Copied from https://github.com/godotengine/godot/blob/c3b0a92c3cd9a219c1b1776b48c147f1d0602f07/methods.py#L1049-L1172
+def show_progress(env):
+ import sys
+ import glob
+ from SCons.Script import Progress, Command, AlwaysBuild
+
+ screen = sys.stdout
+ # Progress reporting is not available in non-TTY environments since it
+ # messes with the output (for example, when writing to a file)
+ show_progress = env["progress"] and sys.stdout.isatty()
+ node_count = 0
+ node_count_max = 0
+ node_count_interval = 1
+ node_count_fname = str(env.Dir("#")) + "/.scons_node_count"
+
+ import time, math
+
+ class cache_progress:
+ # The default is 1 GB cache and 12 hours half life
+ def __init__(self, path=None, limit=1073741824, half_life=43200):
+ self.path = path
+ self.limit = limit
+ self.exponent_scale = math.log(2) / half_life
+ if env["verbose"] and path != None:
+ screen.write(
+ "Current cache limit is {} (used: {})\n".format(
+ self.convert_size(limit), self.convert_size(self.get_size(path))
+ )
+ )
+ self.delete(self.file_list())
+
+ def __call__(self, node, *args, **kw):
+ nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress
+ if show_progress:
+ # Print the progress percentage
+ node_count += node_count_interval
+ if node_count_max > 0 and node_count <= node_count_max:
+ screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max))
+ screen.flush()
+ elif node_count_max > 0 and node_count > node_count_max:
+ screen.write("\r[100%] ")
+ screen.flush()
+ else:
+ screen.write("\r[Initial build] ")
+ screen.flush()
+
+ def delete(self, files):
+ if len(files) == 0:
+ return
+ if env["verbose"]:
+ # Utter something
+ screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file"))
+ [os.remove(f) for f in files]
+
+ def file_list(self):
+ if self.path is None:
+ # Nothing to do
+ return []
+ # Gather a list of (filename, (size, atime)) within the
+ # cache directory
+ file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))]
+ if file_stat == []:
+ # Nothing to do
+ return []
+ # Weight the cache files by size (assumed to be roughly
+ # proportional to the recompilation time) times an exponential
+ # decay since the ctime, and return a list with the entries
+ # (filename, size, weight).
+ current_time = time.time()
+ file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat]
+ # Sort by the most recently accessed files (most sensible to keep) first
+ file_stat.sort(key=lambda x: x[2])
+ # Search for the first entry where the storage limit is
+ # reached
+ sum, mark = 0, None
+ for i, x in enumerate(file_stat):
+ sum += x[1]
+ if sum > self.limit:
+ mark = i
+ break
+ if mark is None:
+ return []
+ else:
+ return [x[0] for x in file_stat[mark:]]
+
+ def convert_size(self, size_bytes):
+ if size_bytes == 0:
+ return "0 bytes"
+ size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+ i = int(math.floor(math.log(size_bytes, 1024)))
+ p = math.pow(1024, i)
+ s = round(size_bytes / p, 2)
+ return "%s %s" % (int(s) if i == 0 else s, size_name[i])
+
+ def get_size(self, start_path="."):
+ total_size = 0
+ for dirpath, dirnames, filenames in os.walk(start_path):
+ for f in filenames:
+ fp = os.path.join(dirpath, f)
+ total_size += os.path.getsize(fp)
+ return total_size
+
+ def progress_finish(target, source, env):
+ nonlocal node_count, progressor
+ try:
+ with open(node_count_fname, "w") as f:
+ f.write("%d\n" % node_count)
+ progressor.delete(progressor.file_list())
+ except Exception:
+ pass
+
+ try:
+ with open(node_count_fname) as f:
+ node_count_max = int(f.readline())
+ except Exception:
+ pass
+
+ cache_directory = os.environ.get("SCONS_CACHE")
+ # Simple cache pruning, attached to SCons' progress callback. Trim the
+ # cache directory to a size not larger than cache_limit.
+ cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024
+ progressor = cache_progress(cache_directory, cache_limit)
+ Progress(progressor, interval=node_count_interval)
+
+ progress_finish_command = Command("progress_finish", [], progress_finish)
+ AlwaysBuild(progress_finish_command)
+
+scons_cache_path = os.environ.get("SCONS_CACHE")
+if scons_cache_path != None:
+ CacheDir(scons_cache_path)
+ print("Scons cache enabled... (path: '" + scons_cache_path + "')")
+
# For the reference:
# - CCFLAGS are compilation flags shared between C and C++
# - CFLAGS are for C-specific compilation flags
@@ -95,4 +231,8 @@ else:
source=sources,
)
+if "env" in locals():
+ # FIXME: This method mixes both cosmetic progress stuff and cache handling...
+ show_progress(env)
+
Default(library)