Merge 0cd1440f13 into be65d37760

LiveRecomp/live_generator.cpp

@@ -1896,3 +1896,29 @@ bool N64Recomp::recompile_function_live(LiveGenerator& generator, const Context&
     return recompile_function_custom(generator, context, function_index, output_file, static_funcs_out, tag_reference_relocs);
 }
 
+N64Recomp::ShimFunction::ShimFunction(recomp_func_ext_t* to_shim, uintptr_t value) {
+    sljit_compiler* compiler = sljit_create_compiler(nullptr);
+
+    // Create the function.
+    sljit_label* func_label = sljit_emit_label(compiler);
+    sljit_emit_enter(compiler, 0, SLJIT_ARGS2V(P_R, P_R), 3, 0, 0);
+
+    // Move the provided value into the third argument.
+    sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, sljit_sw(value));
+
+    // Tail call the provided function.
+    sljit_emit_icall(compiler, SLJIT_CALL | SLJIT_CALL_RETURN, SLJIT_ARGS3V(P, P, W), SLJIT_IMM, sljit_sw(to_shim));
+
+    // Generate the function's code and get the address to the function.
+    code = sljit_generate_code(compiler, 0, nullptr);
+    func = reinterpret_cast<recomp_func_t*>(sljit_get_label_addr(func_label));
+
+    // Cleanup.
+    sljit_free_compiler(compiler);
+}
+
+N64Recomp::ShimFunction::~ShimFunction() {
+    sljit_free_code(code, nullptr);
+    code = nullptr;
+    func = nullptr;
+}

README.md

@@ -52,6 +52,12 @@ RSP microcode can also be recompiled with this tool. Currently there is no suppo
 ## Building
 This project can be built with CMake 3.20 or above and a C++ compiler that supports C++20. This repo uses git submodules, so be sure to clone recursively (`git clone --recurse-submodules`) or initialize submodules recursively after cloning (`git submodule update --init --recursive`). From there, building is identical to any other cmake project, e.g. run `cmake` in the target build folder and point it at the root of this repo, then run `cmake --build .` from that target folder.
 
+## Nix
+If you are using nixos or have [nix installed](https://nixos.org/)
+This project can be built and run using nix flakes, to spawn a shell with both, N64Recomp and RSPRecomp programs available you can perform the command:
+
+`nix shell github:Mr-Wiseguy/N64Recomp`
+
 ## Libraries Used
 * [rabbitizer](https://github.com/Decompollaborate/rabbitizer) for instruction decoding/analysis
 * [ELFIO](https://github.com/serge1/ELFIO) for elf parsing

RecompModTool/main.cpp

@@ -32,6 +32,7 @@ struct ModManifest {
     std::string game_id;
     std::string minimum_recomp_version;
     std::unordered_map<std::string, std::vector<std::string>> native_libraries;
+    std::vector<toml::table> config_options;
     std::vector<std::string> dependencies;
     std::vector<std::string> full_dependency_strings;
 };
@@ -218,6 +219,11 @@ static std::vector<std::filesystem::path> get_toml_path_array(const toml::array&
     return ret;
 }
 
+bool validate_config_option(const toml::table& option) {
+    // TODO config option validation.
+    return true;
+}
+
 ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, const toml::table& manifest_table) {
     ModManifest ret;
 
@@ -317,6 +323,23 @@ ModManifest parse_mod_config_manifest(const std::filesystem::path& basedir, cons
         });
     }
 
+    // Config schema (optional)
+    const toml::array& config_options_array = read_toml_array(manifest_table, "config_options", false);
+    if (!config_options_array.empty()) {
+        ret.config_options.reserve(config_options_array.size());
+        config_options_array.for_each([&ret](const auto& el) {
+            if constexpr (toml::is_table<decltype(el)>) {
+                if (!validate_config_option(el)) {
+                    throw toml::parse_error("Invalid config option", el.source());
+                }
+                ret.config_options.emplace_back(el);
+            }
+            else {
+                throw toml::parse_error("Invalid type for config option", el.source());
+            }
+        });
+    }
+
     return ret;
 }
 
@@ -484,6 +507,14 @@ void write_manifest(const std::filesystem::path& path, const ModManifest& manife
         output_data.emplace("dependencies", string_vector_to_toml(manifest.full_dependency_strings));
     }
 
+    if (!manifest.config_options.empty()) {
+        toml::array options_array{};
+        for (const auto& option : manifest.config_options) {
+            options_array.emplace_back(option);
+        }
+        output_data.emplace("config_schema", toml::table{{"options", std::move(options_array)}});
+    }
+
     toml::json_formatter formatter{output_data, toml::format_flags::indentation | toml::format_flags::indentation};
     std::ofstream output_file(path);
 

flake.lock

@@ -0,0 +1,78 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1715534503,
+        "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "2057814051972fa1453ddfb0d98badbea9b83c06",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,51 @@
+{
+  description = "Tool to statically recompile N64 games into native executables";
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    flake-compat = {
+      url = "github:edolstra/flake-compat";
+      flake = false;
+    };
+  };
+
+  outputs = inputs @ {...}:
+    inputs.flake-utils.lib.eachDefaultSystem
+    (
+      system: let
+        pkgs = import inputs.nixpkgs {
+          inherit system;
+        };
+      in rec {
+        packages = rec {
+          N64Recomp = pkgs.stdenv.mkDerivation {
+            pname = "N64Recomp";
+            version = "1.0.0";
+            src = ./.;
+            nativeBuildInputs = [pkgs.cmake];
+
+            installPhase = ''
+                    mkdir -p $out/bin
+                    cp N64Recomp $out/bin
+                    cp RSPRecomp $out/bin
+                '';
+          };
+          default = N64Recomp;
+        };
+        apps = rec {
+          N64Recomp = {
+            type = "app";
+            program = "${packages.N64Recomp}/bin/N64Recomp";
+          };
+
+          RSPRecomp = {
+            type = "app";
+            program = "${packages.N64Recomp}/bin/RSPRecomp";
+          };
+          default = N64Recomp;
+        };
+      }
+    );
+}

include/recomp.h

@@ -439,7 +439,11 @@ gpr cop0_status_read(recomp_context* ctx);
 void switch_error(const char* func, uint32_t vram, uint32_t jtbl);
 void do_break(uint32_t vram);
 
+// The function signature for all recompiler output functions.
 typedef void (recomp_func_t)(uint8_t* rdram, recomp_context* ctx);
+// The function signature for special functions that need a third argument.
+// These get called via generated shims to allow providing some information about the caller, such as mod id.
+typedef void (recomp_func_ext_t)(uint8_t* rdram, recomp_context* ctx, uintptr_t arg);
 
 recomp_func_t* get_function(int32_t vram);
 

include/recompiler/live_recompiler.h

@@ -141,6 +141,16 @@ namespace N64Recomp {
 
     void live_recompiler_init();
     bool recompile_function_live(LiveGenerator& generator, const Context& context, size_t function_index, std::ostream& output_file, std::span<std::vector<uint32_t>> static_funcs_out, bool tag_reference_relocs);
+
+    class ShimFunction {
+    private:
+        void* code;
+        recomp_func_t* func;
+    public:
+        ShimFunction(recomp_func_ext_t* to_shim, uintptr_t value);
+        ~ShimFunction();
+        recomp_func_t* get_func() { return func; }
+    };
 }
 
 #endif

src/elf.cpp

@@ -104,10 +104,10 @@ bool read_symbols(N64Recomp::Context& context, const ELFIO::elfio& elf_file, ELF
 
                 if (section_index < context.sections.size()) {
                     auto section_offset = value - elf_file.sections[section_index]->get_address();
-                    const uint32_t* words = reinterpret_cast<const uint32_t*>(elf_file.sections[section_index]->get_data() + section_offset);
                     uint32_t vram = static_cast<uint32_t>(value);
                     uint32_t num_instructions = type == ELFIO::STT_FUNC ? size / 4 : 0;
                     uint32_t rom_address = static_cast<uint32_t>(section_offset + section.rom_addr);
+                    const uint32_t* words = reinterpret_cast<const uint32_t*>(context.rom.data() + rom_address);
 
                     section.function_addrs.push_back(vram);
                     context.functions_by_vram[vram].push_back(context.functions.size());
@@ -551,6 +551,36 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP
                     return a.address < b.address;
                 }
             );
+
+            // Patch the ROM word for HI16 and LO16 reference symbol relocs to non-relocatable sections.
+            for (size_t i = 0; i < section_out.relocs.size(); i++) {
+                auto& reloc = section_out.relocs[i];
+                if (reloc.reference_symbol && (reloc.type == N64Recomp::RelocType::R_MIPS_HI16 || reloc.type == N64Recomp::RelocType::R_MIPS_LO16)) {
+                    bool target_section_relocatable = context.is_reference_section_relocatable(reloc.target_section);
+                    if (!target_section_relocatable) {
+                        uint32_t reloc_rom_addr = reloc.address - section_out.ram_addr + section_out.rom_addr;
+                        uint32_t reloc_rom_word = byteswap(*reinterpret_cast<const uint32_t*>(context.rom.data() + reloc_rom_addr));
+
+                        uint32_t ref_section_vram = context.get_reference_section_vram(reloc.target_section);
+                        uint32_t full_immediate = reloc.target_section_offset + ref_section_vram;
+
+                        uint32_t imm;
+
+                        if (reloc.type == N64Recomp::RelocType::R_MIPS_HI16) {
+                            imm = (full_immediate >> 16) + ((full_immediate >> 15) & 1);
+                        }
+                        else {
+                            imm = full_immediate & 0xFFFF;
+                        }
+
+                        *reinterpret_cast<uint32_t*>(context.rom.data() + reloc_rom_addr) = byteswap(reloc_rom_word | imm);
+                        // Remove the reloc by setting it to a type of NONE.
+                        reloc.type = N64Recomp::RelocType::R_MIPS_NONE;
+                        reloc.reference_symbol = false;
+                        reloc.symbol_index = (uint32_t)-1;
+                    }
+                }
+            }
         }
     }
 

src/recompilation.cpp

@@ -184,19 +184,10 @@ bool process_instruction(GeneratorType& generator, const N64Recomp::Context& con
                     reloc_reference_symbol = reloc.symbol_index;
                     // Don't try to relocate special section symbols.
                     if (context.is_regular_reference_section(reloc.target_section) || reloc_section == N64Recomp::SectionAbsolute) {
+                        // TODO this may not be needed anymore as HI16/LO16 relocs to non-relocatable sections is handled directly in elf parsing.
                         bool ref_section_relocatable = context.is_reference_section_relocatable(reloc.target_section);
                         // Resolve HI16 and LO16 reference symbol relocs to non-relocatable sections by patching the instruction immediate.
                         if (!ref_section_relocatable && (reloc_type == N64Recomp::RelocType::R_MIPS_HI16 || reloc_type == N64Recomp::RelocType::R_MIPS_LO16)) {
-                            uint32_t ref_section_vram = context.get_reference_section_vram(reloc.target_section);
-                            uint32_t full_immediate = reloc.target_section_offset + ref_section_vram;
-
-                            if (reloc_type == N64Recomp::RelocType::R_MIPS_HI16) {
-                                imm = (full_immediate >> 16) + ((full_immediate >> 15) & 1);
-                            }
-                            else if (reloc_type == N64Recomp::RelocType::R_MIPS_LO16) {
-                                imm = full_immediate & 0xFFFF;
-                            }
-
                             // The reloc has been processed, so set it to none to prevent it getting processed a second time during instruction code generation.
                             reloc_type = N64Recomp::RelocType::R_MIPS_NONE;
                             reloc_reference_symbol = (size_t)-1;