Faster builds

backend/app/tasks/deploy_frame.py

@@ -57,7 +57,7 @@ def install_if_necessary(package: str, raise_on_error = True) -> int:
                 uname_output = []
                 exec_command(frame, ssh, "uname -m", uname_output)
                 arch = "".join(uname_output).strip()
-                if arch == "aarch64":
+                if arch == "aarch64" or arch == "arm64":
                     cpu = "arm64"
                 elif arch == "armv6l" or arch == "armv7l":
                     cpu = "arm"
@@ -104,7 +104,7 @@ def install_if_necessary(package: str, raise_on_error = True) -> int:
                     log(id, "stdout", f"> add /srv/frameos/build/build_{build_id}.tar.gz")
                     scp.put(archive_path, f"/srv/frameos/build/build_{build_id}.tar.gz")
                     exec_command(frame, ssh, f"cd /srv/frameos/build && tar -xzf build_{build_id}.tar.gz && rm build_{build_id}.tar.gz")
-                    exec_command(frame, ssh, f"cd /srv/frameos/build/build_{build_id} && sh ./compile_frameos.sh")
+                    exec_command(frame, ssh, f"cd /srv/frameos/build/build_{build_id} && make -j$(nproc)")
                     exec_command(frame, ssh, f"mkdir -p /srv/frameos/releases/release_{build_id}")
                     exec_command(frame, ssh, f"cp /srv/frameos/build/build_{build_id}/frameos /srv/frameos/releases/release_{build_id}/frameos")
                     log(id, "stdout", f"> add /srv/frameos/releases/release_{build_id}/frame.json")
@@ -295,43 +295,29 @@ def create_local_build_archive(frame: Frame, build_dir: str, build_id: str, nim_
             for file in files:
                 shutil.copy(os.path.join(source_dir, "src", "drivers", "waveshare", "ePaper", file), os.path.join(build_dir, file))
 
-    # Update the compilation script for verbose output
-    script_path = os.path.join(build_dir, "compile_frameos.sh")
-    log(frame.id, "stdout", f"Cleaning build script at {script_path}")
-    with open(script_path, "r") as file:
-        lines = file.readlines()
-    with open(script_path, "w") as file:
-        file.write("#!/bin/sh\n")
-        file.write("set -eu\n")
-        file.write("start_time=$(date +%s)\n")
-        file.write("mkdir -p ../cache\n") # make sure we have the cache folder
-        file.write("cached_files_count=0\n")  # Initialize cached files counter
-        for i, line in enumerate(lines):
-            if line.startswith("gcc -c") and line.strip().endswith(".c"):
-                source_file = line.split(' ')[-1].strip()
-                o_file = line.split(' ')[-2].strip()
-                source_cleaned = '/'.join(source_file.split('@s')[-3:]).replace('@m', './')
-
-                # take the md5sum of the source file <source>.c
-                file.write(f"md5sum={compile_line_md5(line)}$(md5sum {source_file} | awk '{{print $1}}')\n")
-                # check if there's a file in the cache folder called <md5sum>.c.o
-                file.write(f"if [ -f ../cache/${{md5sum}}.{cpu}.c.o ]; then\n")
-                # if there is, make a symlink to the build folder with the name <source>.o
-                file.write("    cached_files_count=$((cached_files_count + 1))\n")
-                file.write(f"    ln -s ../cache/${{md5sum}}.{cpu}.c.o {o_file}\n")
-                file.write("else\n")
-                # if not, run the command in "line" and then copy the <source>.c.o into the build folder as <md5sum>.c.o
-                file.write(f"    echo [{i + 1}/{len(lines)}] Compiling on device: {source_cleaned}\n")
-                file.write(f"    {line.strip()}\n")
-                file.write(f"    cp {o_file} ../cache/${{md5sum}}.{cpu}.c.o\n")
-                file.write("fi\n")
-            else:
-                file.write(f"echo [{i + 1}/{len(lines)}] Compiling on device: frameos\n")
-                file.write(line)
-        file.write("echo \"Used $cached_files_count cached files\"\n")
-        file.write("end_time=$(date +%s)\n")
-        file.write("duration=$((end_time - start_time))\n")
-        file.write("echo \"Compiled in $duration seconds\"\n")
+    # Create Makefile
+    with open(os.path.join(build_dir, "Makefile"), "w") as file:
+        # Read the compilation flags from the generated script
+        script_path = os.path.join(build_dir, "compile_frameos.sh")
+        linker_flags = ["-pthread", "-lm", "-lrt", "-ldl"]
+        compiler_flags = []
+        with open(script_path, "r") as script:
+            lines = script.readlines()
+        for line in lines:
+            if " -o frameos " in line and " -l" in line:
+                linker_flags = [flag.strip() for flag in line.split(' ') if flag.startswith('-') and flag != '-o']
+            elif " -c " in line and len(compiler_flags) == 0:
+                compiler_flags = [flag for flag in line.split(' ') if flag.startswith('-') and not flag.startswith('-I') and not flag in ['-o', '-c']]
+
+        # Read the Makefile from ../frameos/tools/nimc.Makefile
+        with open(os.path.join(source_dir, "tools", "nimc.Makefile"), "r") as makefile:
+            lines = makefile.readlines()
+        for line in lines:
+            if line.startswith("LIBS = "):
+                line = "LIBS = -L. " + (" ".join(linker_flags)) + "\n"
+            if line.startswith("CFLAGS = "):
+                line = "CFLAGS = " + (" ".join([f for f in compiler_flags if f != '-c'])) + "\n"
+            file.write(line)
 
     # 7. Zip it up "(cd tmp && tar -czf ./build_1.tar.gz build_1)"
     archive_path = os.path.join(temp_dir, f"build_{build_id}.tar.gz")

frameos/tools/nimc.Makefile

@@ -0,0 +1,39 @@
+# This Makefile is used for compiling C sources generated by Nim
+CC ?= gcc
+
+SOURCES := $(shell ls -S *.c)
+OBJECTS = $(SOURCES:.c=.o)
+TOTAL = $(words $(SOURCES))
+EXECUTABLE = frameos
+LIBS = -pthread -lm -lm -lrt -ldl
+CFLAGS = -w -fmax-errors=3 -pthread -O3 -fno-strict-aliasing -fno-ident -fno-math-errno
+
+all: $(EXECUTABLE)
+
+$(EXECUTABLE): $(OBJECTS)
+	@echo "Linking frameos"
+	@$(CC) -o $(EXECUTABLE) $(OBJECTS) $(LIBS)
+
+clean:
+	rm -f *.o $(EXECUTABLE)
+
+pre-build:
+	mkdir -p ../cache
+
+$(OBJECTS): pre-build
+
+%.o: %.c
+	@if [ ! -e $@ ]; then \
+		md5sum=$$(md5sum $< | awk '{print $1}'); \
+		file=$$(echo '$<' | sed 's/@s/\//g' | sed 's/@m//g' | sed 's/.*nimble\/pkgs2\/\(.*\)/\1/' | sed 's/.*\/\(nim\/lib\/.*\)/\1/'); \
+		cache_obj=../cache/$$md5sum.c.o; \
+		if [ -f "$$cache_obj" ]; then \
+			ln -s "$$cache_obj" $@; \
+		else \
+			$(CC) -c $(CFLAGS) $< -o $@; \
+			cp $@ "$$cache_obj"; \
+			echo "[$$(ls *.o | wc -l)/$(TOTAL)] $$file"; \
+		fi; \
+	fi
+
+.PHONY: all clean pre-build