[WIP] Add Base.build_sysimg()

Make.inc

@@ -266,17 +266,6 @@ LD := link
 endif
 RANLIB := $(CROSS_COMPILE)ranlib
 
-ifeq ($(JULIA_CPU_TARGET),native)
-  JCPPFLAGS += -DJULIA_TARGET_NATIVE
-else ifeq ($(JULIA_CPU_TARGET),core2)
-  JCPPFLAGS += -DJULIA_TARGET_CORE2
-else ifeq ($(JULIA_CPU_TARGET),i386)
-  JCPPFLAGS += -DJULIA_TARGET_I386
-else
-  $(error Unknown cpu target architecture)
-endif
-
-
 # Calculate relative paths to libdir and private_libdir
 build_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_libdir))
 libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(libdir))

Makefile

@@ -81,11 +81,11 @@ endif
 
 $(build_private_libdir)/sys0.o:
 	@$(QUIET_JULIA) cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl
+	$(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl
 
 $(build_private_libdir)/sys.o: VERSION base/*.jl base/pkg/*.jl base/linalg/*.jl base/sparse/*.jl $(build_datarootdir)/julia/helpdb.jl $(build_datarootdir)/man/man1/julia.1 $(build_private_libdir)/sys0.$(SHLIB_EXT)
 	@$(QUIET_JULIA) cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \
+	$(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys) \
 		-J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \
 		|| (echo "*** This error is usually fixed by running 'make clean'. If the error persists, try 'make cleanall'. ***" && false)
 

base/interactiveutil.jl

@@ -329,4 +329,4 @@ end
 function download(url::String)
     filename = tempname()
     download(url, filename)
-end
+end

contrib/build_sysimg.jl

@@ -0,0 +1,77 @@
+# By default, put the system image next to libjulia
+build_sysimg(;force=false, cpu_target="native") = build_sysimg(joinpath(dirname(Sys.dlpath("libjulia")),"sys"), force=force, cpu_target=cpu_target)
+
+# Build a system image binary at sysimg_path.dlext.  If a system image is already loaded, error out, or continue if force = true
+function build_sysimg(sysimg_path; force=false, cpu_target="native")
+    # Unless force == true, quit out if a sysimg is already loadable
+    sysimg = dlopen_e("sys")
+    if !force && sysimg != C_NULL
+        println("System image already loaded at $(Sys.dlpath(sysimg)), pass \"force=true\" to override")
+        return;
+    end
+
+    # Enter base/ and setup some useful paths
+    base_dir = dirname(Base.find_source_file("sysimg.jl"))
+    cd(base_dir) do
+        julia = joinpath(JULIA_HOME, "julia")
+        julia_libdir = dirname(Sys.dlpath("libjulia"))
+        ld = find_system_linker()
+
+        # Ensure we have write-permissions to wherever we're trying to write to
+        if !success(`touch $sysimg_path.$(Sys.dlext)`)
+            error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)")
+        end
+
+        # Start by building sys0.{ji,o}
+        sys0_path = joinpath(dirname(sysimg_path), "sys0")
+        println("Building sys0.o...")
+        println("$julia -C $cpu_target --build $sys0_path sysimg.jl")
+        run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`)
+
+        # Bootstrap off of that to create sys.{ji,o}
+        println("Building sys.o...")
+        println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl")
+        run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`)
+
+        # Link sys.o into sys.$(dlext)
+        FLAGS = ["-L$julia_libdir"]
+        if OS_NAME == :Darwin
+            push!(FLAGS, "-dylib")
+            push!(FLAGS, "-undefined")
+            push!(FLAGS, "dynamic_lookup")
+        else
+            push!(FLAGS, "--unresolved-symbols")
+            push!(FLAGS, "ignore-all")
+        end
+        @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"])
+
+        println("Linking sys.$(Sys.dlext)")
+        run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`)
+    end
+end
+
+# Search for a linker to link sys.o into sys.dl_ext.  Honor LD environment variable, otherwise search for something we know works
+function find_system_linker()
+    if haskey( ENV, "LD" )
+        if !success(`which $(ENV["LD"])`)
+            warn("Using linker override $(ENV["LD"]), but unable to find `$(ENV["LD"])`")
+        end
+        return ENV["LD"]
+    end
+
+    poss_linkers = ["ld", "link"]
+
+    for linker in poss_linkers
+        try
+            if success(`which $linker`)
+                return linker
+            end
+        end
+    end
+
+    error( "No supported linker found (tried $(join(poss_linkers, ", "))), override with LD environment variable!" )
+end
+
+if !isinteractive()
+    build_sysimg()
+end

src/cgutils.cpp

@@ -240,7 +240,7 @@ static void jl_gen_llvm_gv_array()
             ConstantInt::get(T_size,globalUnique+1),
             "jl_globalUnique");
 
-    Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_cpu_string);
+    Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_compileropts.cpu_target);
     new GlobalVariable(*jl_Module,
                        feature_string->getType(),
                        true,
@@ -249,7 +249,7 @@ static void jl_gen_llvm_gv_array()
                        "jl_sysimg_cpu_target");
 
     // For native also store the cpuid
-    if (strcmp(jl_cpu_string,"native") == 0) {
+    if (strcmp(jl_compileropts.cpu_target, "native") == 0) {
         uint32_t info[4];
 
         jl_cpuid((int32_t*)info, 1);

src/codegen.cpp

@@ -493,16 +493,6 @@ static Type *NoopType;
 // --- utilities ---
 
 extern "C" {
-#if defined(JULIA_TARGET_CORE2)
-const char *jl_cpu_string = "core2";
-#elif defined(JULIA_TARGET_I386)
-const char *jl_cpu_string = "i386";
-#elif defined(JULIA_TARGET_NATIVE)
-const char *jl_cpu_string = "native";
-#else
-#error "Must select julia cpu target"
-#endif
-
     int globalUnique = 0;
 }
 
@@ -4413,17 +4403,18 @@ extern "C" void jl_init_codegen(void)
     const char *mattr[] = {"-bmi2", "-avx2"};
     std::vector<std::string> attrvec (mattr, mattr+2);
 #endif
+
+    const char * cpu_target = jl_compileropts.cpu_target;
+    if (strcmp(cpu_target, "native") == 0)
+        cpu_target = "";
+
     EngineBuilder eb = EngineBuilder(engine_module)
         .setEngineKind(EngineKind::JIT)
 #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) && !defined(USE_MCJIT)
         .setJITMemoryManager(new JITMemoryManagerWin())
 #endif
         .setTargetOptions(options)
-#if defined(JULIA_TARGET_NATIVE)
-        .setMCPU("")
-#else
-        .setMCPU(jl_cpu_string)
-#endif
+        .setMCPU(cpu_target)
 #ifdef USE_MCJIT
         .setUseMCJIT(true)
         .setMAttrs(attrvec);
@@ -4440,7 +4431,7 @@ extern "C" void jl_init_codegen(void)
 #if defined(JULIA_TARGET_NATIVE)
     jl_TargetMachine = eb.selectTarget(TheTriple,"","",MAttrs);
 #else 
-    jl_TargetMachine = eb.selectTarget(TheTriple,"",jl_cpu_string,MAttrs);
+    jl_TargetMachine = eb.selectTarget(TheTriple,"",cpu_target,MAttrs);
 #endif
     assert(jl_TargetMachine);
     jl_ExecutionEngine = eb.create(jl_TargetMachine);

src/dump.c

@@ -100,7 +100,6 @@ jl_value_t ***sysimg_gvars = NULL;
 
 extern int globalUnique;
 extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType);
-extern const char *jl_cpu_string;
 uv_lib_t *jl_sysimg_handle = NULL;
 char *jl_sysimage_name = NULL;
 
@@ -114,25 +113,25 @@ static void jl_load_sysimg_so(char *fname)
         sysimg_gvars = (jl_value_t***)jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars");
         globalUnique = *(size_t*)jl_dlsym(jl_sysimg_handle, "jl_globalUnique");
         const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target");
-        if (strcmp(cpu_target,jl_cpu_string) != 0)
+        if (strcmp(cpu_target, jl_compileropts.cpu_target) != 0)
             jl_error("Julia and the system image were compiled for different architectures.\n"
-                     "Please delete or regenerate sys.{so,dll,dylib}.");
+                     "Please delete or regenerate sys.{so,dll,dylib}.\n");
         uint32_t info[4];
         jl_cpuid((int32_t*)info, 1);
         if (strcmp(cpu_target, "native") == 0) {
             uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_cpuid");
             if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32)))
-                jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.");
+                jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.\n");
         }
         else if (strcmp(cpu_target,"core2") == 0) {
             int HasSSSE3 = (info[3] & 1<<9);
             if (!HasSSSE3)
                 jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n"
-                         "Please delete or regenerate sys.{so,dll,dylib}.");
+                         "Please delete or regenerate sys.{so,dll,dylib}.\n");
         }
         else if (strcmp(cpu_target,"i386") != 0) {
             jl_error("System image has unknown target cpu architecture.\n"
-                     "Please delete or regenerate sys.{so,dll,dylib}.");
+                     "Please delete or regenerate sys.{so,dll,dylib}.\n");
         }
         jl_sysimage_name = strdup(fname);
     }
@@ -1033,6 +1032,32 @@ extern void jl_get_builtin_hooks(void);
 extern void jl_get_system_hooks(void);
 extern void jl_get_uv_hooks();
 
+// Takes in a path of the form "usr/lib/julia/sys.ji", such as passed in to jl_restore_system_image()
+DLLEXPORT
+const char * jl_get_system_image_cpu_target(char *fname)
+{
+    // If passed NULL, don't even bother
+    if (!fname)
+        return NULL;
+
+    // First, get "sys" from "sys.ji"
+    char *fname_shlib = (char*)alloca(strlen(fname));
+    strcpy(fname_shlib, fname);
+    char *fname_shlib_dot = strrchr(fname_shlib, '.');
+    if (fname_shlib_dot != NULL)
+        *fname_shlib_dot = 0;
+
+    // Get handle to sys.so
+    uv_lib_t * sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL);
+
+    // Return jl_sysimg_cpu_target if we can
+    if (sysimg_handle)
+        return (const char *)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target");
+
+    // If something goes wrong, return NULL
+    return NULL;
+}
+
 DLLEXPORT
 void jl_restore_system_image(char *fname)
 {

src/init.c

@@ -65,10 +65,12 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE);
 #endif
 
 char *julia_home = NULL;
-jl_compileropts_t jl_compileropts = { NULL, // build_path
-                                      0,    // code_coverage
+jl_compileropts_t jl_compileropts = { NULL,     // build_path
+                                      NULL,     // cpu_target ("native", "core2", etc...)
+                                      0,        // code_coverage
                                       JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT,
-                                      0     // int32_literals
+                                      0,        // int32_literals
+
 };
 
 int jl_boot_file_loaded = 0;
@@ -728,6 +730,14 @@ void julia_init(char *imageFile)
     jl_init_frontend();
     jl_init_types();
     jl_init_tasks(jl_stack_lo, jl_stack_hi-jl_stack_lo);
+
+    // If we are able to load the imageFile and get a cpu_target, use that unless user has overridden
+    if (jl_compileropts.cpu_target == NULL) {
+        const char * sysimg_cpu_target = jl_get_system_image_cpu_target(imageFile);
+
+        // If we can't load anything from the sysimg, default to native
+        jl_compileropts.cpu_target = sysimg_cpu_target ? (char *)sysimg_cpu_target : "native";
+    }
     jl_init_codegen();
     jl_an_empty_cell = (jl_value_t*)jl_alloc_cell_1d(0);
 

src/julia.h

@@ -871,6 +871,7 @@ DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_pat
 DLLEXPORT int jl_is_initialized(void);
 DLLEXPORT extern char *julia_home;
 
+DLLEXPORT const char * jl_get_system_image_cpu_target(char *fname);
 DLLEXPORT void jl_save_system_image(char *fname);
 DLLEXPORT void jl_restore_system_image(char *fname);
 void jl_init_restored_modules();
@@ -1310,6 +1311,7 @@ void jl_print_gc_stats(JL_STREAM *s);
 
 typedef struct {
     char *build_path;
+    char *cpu_target;
     int8_t code_coverage;
     int8_t check_bounds;
     int int_literals;

ui/repl.c

@@ -59,11 +59,12 @@ static const char *opts =
     " -q, --quiet              Quiet startup without banner\n"
     " -H, --home <dir>         Set location of julia executable\n\n"
 
-    " -e, --eval <expr>        Evaluate <expr>\n"
-    " -E, --print <expr>       Evaluate and show <expr>\n"
-    " -P, --post-boot <expr>   Evaluate <expr> right after boot\n"
-    " -L, --load <file>        Load <file> right after boot on all processors\n"
-    " -J, --sysimage <file>    Start up with the given system image file\n\n"
+    " -e --eval <expr>         Evaluate <expr>\n"
+    " -E --print <expr>        Evaluate and show <expr>\n"
+    " -P --post-boot <expr>    Evaluate <expr> right after boot\n"
+    " -L --load <file>         Load <file> right after boot on all processors\n"
+    " -J --sysimage <file>     Start up with the given system image file\n"
+    " -C --cpu-target <target> Limit usage of cpu features up to <target>\n\n"
 
     " -p <n>                   Run n local processes\n"
     " --machinefile <file>     Run processes on hosts listed in <file>\n\n"
@@ -80,7 +81,7 @@ static const char *opts =
 
 void parse_opts(int *argcp, char ***argvp)
 {
-    static char* shortopts = "+H:T:hJ:";
+    static char* shortopts = "+H:hJ:C:";
     static struct option longopts[] = {
         { "home",          required_argument, 0, 'H' },
         { "tab",           required_argument, 0, 'T' },
@@ -89,6 +90,7 @@ void parse_opts(int *argcp, char ***argvp)
         { "help",          no_argument,       0, 'h' },
         { "sysimage",      required_argument, 0, 'J' },
         { "code-coverage", no_argument,       &codecov, 1 },
+        { "cpu-target",    required_argument, 0, 'C' },
         { "check-bounds",  required_argument, 0, 300 },
         { "int-literals",  required_argument, 0, 301 },
         { 0, 0, 0, 0 }
@@ -119,6 +121,15 @@ void parse_opts(int *argcp, char ***argvp)
             image_file = strdup(optarg);
             imagepathspecified = 1;
             break;
+        case 'C':
+            if( strcmp(optarg, "native") == 0 || strcmp(optarg, "i386") == 0 || strcmp(optarg, "core2") == 0 ) {
+                jl_compileropts.cpu_target = strdup(optarg);
+            }
+            else {
+                ios_printf(ios_stderr, "julia: invalid cpu-target \"%s\" specified\n", optarg );
+                exit(1);
+            }
+            break;
         case 'h':
             printf("%s%s", usage, opts);
             exit(0);