Improved Lightrec Debug

Makefile

@@ -118,13 +118,23 @@ endif
 ifeq "$(DYNAREC)" "lightrec"
 CFLAGS += -Ideps/lightning/include -Ideps/lightrec -Iinclude/lightning -Iinclude/lightrec \
 		  -DLIGHTREC -DLIGHTREC_STATIC
+ifeq ($(LIGHTREC_DEBUG),1)
+deps/lightrec/%.o: CFLAGS += -DLOG_LEVEL=DEBUG_L
+libpcsxcore/lightrec/plugin.o: CFLAGS += -DLIGHTREC_DEBUG=1
+frontend/main.o: CFLAGS += -DLIGHTREC_DEBUG=1
+deps/lightning/%.o: CFLAGS += -DDISASSEMBLER=1 -DBINUTILS_2_38=1 -DBINUTILS_2_29=1 \
+	-DHAVE_DISASSEMBLE_INIT_FOR_TARGET=1 -DPACKAGE_VERSION=1
+LDFLAGS += -lopcodes -lbfd
+endif
 LIGHTREC_CUSTOM_MAP ?= 0
 LIGHTREC_CUSTOM_MAP_OBJ ?= libpcsxcore/lightrec/mem.o
 LIGHTREC_THREADED_COMPILER ?= 0
 LIGHTREC_CODE_INV ?= 0
 CFLAGS += -DLIGHTREC_CUSTOM_MAP=$(LIGHTREC_CUSTOM_MAP) \
 	  -DLIGHTREC_CODE_INV=$(LIGHTREC_CODE_INV) \
-	  -DLIGHTREC_ENABLE_THREADED_COMPILER=$(LIGHTREC_THREADED_COMPILER)
+	  -DLIGHTREC_ENABLE_THREADED_COMPILER=$(LIGHTREC_THREADED_COMPILER) \
+	  -DLIGHTREC_ENABLE_DISASSEMBLER=$(or $(LIGHTREC_DEBUG),0) \
+	  -DLIGHTREC_NO_DEBUG=$(if $(LIGHTREC_DEBUG),0,1)
 ifeq ($(LIGHTREC_CUSTOM_MAP),1)
 LDLIBS += -lrt
 OBJS += $(LIGHTREC_CUSTOM_MAP_OBJ)

frontend/main.c

@@ -737,7 +737,9 @@ int main(int argc, char *argv[])
 	else
 		menu_loop();
 
+#ifndef LIGHTREC_DEBUG
 	pl_start_watchdog();
+#endif
 
 	while (!g_emu_want_quit)
 	{

include/lightrec/lightrec-config.h

@@ -8,20 +8,20 @@
 
 #define ENABLE_THREADED_COMPILER LIGHTREC_ENABLE_THREADED_COMPILER
 #define ENABLE_FIRST_PASS 1
-#define ENABLE_DISASSEMBLER 0
+#define ENABLE_DISASSEMBLER LIGHTREC_ENABLE_DISASSEMBLER
 #define ENABLE_CODE_BUFFER 1
 
 #define HAS_DEFAULT_ELM 1
 
 #define OPT_REMOVE_DIV_BY_ZERO_SEQ 1
-#define OPT_REPLACE_MEMSET 1
+#define OPT_REPLACE_MEMSET LIGHTREC_NO_DEBUG
 #define OPT_DETECT_IMPOSSIBLE_BRANCHES 1
 #define OPT_HANDLE_LOAD_DELAYS 1
 #define OPT_TRANSFORM_OPS 1
 #define OPT_LOCAL_BRANCHES 1
 #define OPT_SWITCH_DELAY_SLOTS 1
 #define OPT_FLAG_IO 1
-#define OPT_FLAG_MULT_DIV 1
+#define OPT_FLAG_MULT_DIV LIGHTREC_NO_DEBUG
 #define OPT_EARLY_UNLOAD 1
 #define OPT_PRELOAD_PC 1
 

jni/Android.mk

@@ -154,6 +154,8 @@ else
 endif
   COREFLAGS   += -DLIGHTREC_CUSTOM_MAP=$(LIGHTREC_CUSTOM_MAP)
   COREFLAGS   += -DLIGHTREC_ENABLE_THREADED_COMPILER=$(LIGHTREC_THREADED_COMPILER)
+  COREFLAGS   += -DLIGHTREC_ENABLE_DISASSEMBLER=$(or $(LIGHTREC_DEBUG),0)
+  COREFLAGS   += -DLIGHTREC_NO_DEBUG=$(if $(LIGHTREC_DEBUG),0,1)
 
 ifeq ($(HAVE_ARI64),1)
   SOURCES_C   += $(DYNAREC_DIR)/new_dynarec.c \

libpcsxcore/lightrec/big_ass_debugger.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+from time import sleep
+from sys import argv
+from os import environ
+import subprocess
+
+def get_next_line(p):
+	line = ""
+
+	while line[0:5] != "CYCLE":
+		line = p.readline().decode()
+
+		if (len(line) == 0):
+			sleep(0.001)
+		elif line[0:5] != "CYCLE":
+			print(line[:-1])
+
+	return line
+
+def print_differences(inter, dynarec):
+	inter_array = inter.split(" ")
+	inter_dict = dict(zip(inter_array[::2], inter_array[1::2]))
+	dynarec_array = dynarec.split(" ")
+	dynarec_dict = dict(zip(dynarec_array[::2], dynarec_array[1::2]))
+
+	diff = dict([(k, (inter_dict[k], dynarec_dict[k])) for k in inter_dict.keys() if inter_dict[k] != dynarec_dict[k]])
+
+	print("\nDifferences:")
+	print("{:15}{:15}{:15}".format("", "Interpreter", "Dynarec"))
+	for k in diff:
+		print("{:15}{:15}{:15}".format(k, diff[k][0], diff[k][1]))
+
+def print_mismatch(inter, dynarec, oldline):
+	print("\nMismatch!")
+	print(inter + " - Interpreter")
+	print(dynarec + " - Dynarec")
+	print("State before the mismatch:")
+	print(oldline)
+	print_differences(inter, dynarec)
+
+def read_loop(p1, p2):
+	oldline = ""
+
+	while True:
+		line1 = get_next_line(p1)
+		line2 = get_next_line(p2)
+
+		if line1 != line2:
+			# TODO: Proper matching
+
+			# Lightrec might be lagging behind
+			#if line1[0:16] != line2[0:16]:
+			if line1[6:16] != line2[6:16]:
+				cycle1 = int(line1[6:16], 16)
+				cycle2 = int(line2[6:16], 16)
+
+				if cycle1 < cycle2:
+					print(line2[:-1] + " - Dynarec")
+
+					while cycle1 < cycle2:
+						print(line1[:-1] + " - Interpreter lagging behind")
+						print_differences(line1[:-1], line2[:-1])
+						line1 = get_next_line(p1)
+						cycle1 = int(line1[6:16], 16)
+
+					while cycle1 > cycle2:
+						print(line2[:-1] + " - Dynarec lagging behind")
+						print_differences(line1[:-1], line2[:-1])
+						line2 = get_next_line(p2)
+						cycle2 = int(line2[6:16], 16)
+
+					if line1 != line2:
+						print_mismatch(line1[:-1], line2[:-1], oldline)
+						break
+
+				if cycle2 < cycle1:
+					print(line1[:-1] + " - Interpreter")
+
+					while cycle1 > cycle2:
+						print(line2[:-1] + " - Dynarec lagging behind")
+						print_differences(line1[:-1], line2[:-1])
+						line2 = get_next_line(p2)
+						cycle2 = int(line2[6:16], 16)
+
+					while cycle1 < cycle2:
+						print(line1[:-1] + " - Interpreter lagging behind")
+						print_differences(line1[:-1], line2[:-1])
+						line1 = get_next_line(p1)
+						cycle1 = int(line1[6:16], 16)
+
+					if line1 != line2:
+						print_mismatch(line1[:-1], line2[:-1], oldline)
+						break
+
+				if line1 == line2:
+					oldline = line1[:-1]
+					print(oldline[:16] + " - Match")
+				continue
+
+			print_mismatch(line1[:-1], line2[:-1], oldline)
+			break
+		else:
+			oldline = line1[:-1]
+
+def main():
+	with subprocess.Popen(['./pcsx'] + argv[1:], env={ **environ, 'LIGHTREC_DEBUG': '1', 'LIGHTREC_INTERPRETER': '1' }, stdout=subprocess.PIPE, bufsize=1) as fifo_int:
+		with subprocess.Popen(['./pcsx'] + argv[1:], env={ **environ, 'LIGHTREC_DEBUG': '1' }, stdout=subprocess.PIPE, bufsize=1) as fifo_jit:
+			read_loop(fifo_int.stdout, fifo_jit.stdout)
+
+if __name__ == '__main__':
+	main()

libpcsxcore/lightrec/plugin.c

@@ -72,6 +72,9 @@ static bool use_lightrec_interpreter;
 static bool block_stepping;
 //static bool use_pcsx_interpreter;
 #define use_pcsx_interpreter 0
+static bool ram_disabled;
+static bool lightrec_debug, lightrec_very_debug;
+static u32 lightrec_begin_cycles;
 
 extern u32 lightrec_hacks;
 
@@ -340,6 +343,8 @@ static void lightrec_enable_ram(struct lightrec_state *state, bool enable)
 		memcpy(psxM, cache_buf, sizeof(cache_buf));
 	else
 		memcpy(cache_buf, psxM, sizeof(cache_buf));
+
+	ram_disabled = !enable;
 }
 
 static bool lightrec_can_hw_direct(u32 kaddr, bool is_write, u8 size)
@@ -465,6 +470,16 @@ static int lightrec_plugin_init(void)
 
 	use_lightrec_interpreter = !!getenv("LIGHTREC_INTERPRETER");
 
+#ifdef LIGHTREC_DEBUG
+	char *cycles = getenv("LIGHTREC_BEGIN_CYCLES");
+
+	lightrec_very_debug = !!getenv("LIGHTREC_VERY_DEBUG");
+	lightrec_debug = lightrec_very_debug || !!getenv("LIGHTREC_DEBUG");
+
+	if (cycles)
+		lightrec_begin_cycles = (unsigned int) strtol(cycles, NULL, 0);
+#endif
+
 	lightrec_state = lightrec_init(LIGHTREC_PROG_NAME,
 			lightrec_map, ARRAY_SIZE(lightrec_map),
 			&lightrec_ops);
@@ -481,13 +496,112 @@ static int lightrec_plugin_init(void)
 	return 0;
 }
 
+static u32 do_calculate_hash(const void *buffer, u32 count, u32 needle, bool le)
+{
+	unsigned int i;
+	const u32 *data = (const u32 *) buffer;
+	u32 hash = needle;
+
+	count /= 4;
+	for(i = 0; i < count; ++i) {
+		hash += le ? LE32TOH(data[i]) : data[i];
+		hash += (hash << 10);
+		hash ^= (hash >> 6);
+	}
+
+	hash += (hash << 3);
+	hash ^= (hash >> 11);
+	hash += (hash << 15);
+
+	return hash;
+}
+
+static u32 hash_calculate_le(const void *buffer, u32 count)
+{
+	return do_calculate_hash(buffer, count, 0xffffffff, true);
+}
+
+u32 hash_calculate(const void *buffer, u32 count)
+{
+	return do_calculate_hash(buffer, count, 0xffffffff, false);
+}
+
+static u32 hash_calculate_ram(const void *buffer, u32 ram_size)
+{
+	u32 hash;
+
+	if (ram_disabled)
+		hash = hash_calculate_le(cache_buf, sizeof(cache_buf));
+	else
+		hash = hash_calculate_le(buffer, sizeof(cache_buf));
+
+	return do_calculate_hash(buffer + sizeof(cache_buf),
+				 ram_size - sizeof(cache_buf),
+				 hash, true);
+}
+
+static const char * const mips_regs[] = {
+	"zero",
+	"at",
+	"v0", "v1",
+	"a0", "a1", "a2", "a3",
+	"t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
+	"s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+	"t8", "t9",
+	"k0", "k1",
+	"gp", "sp", "fp", "ra",
+	"lo", "hi",
+};
+
+static void print_for_big_ass_debugger(void)
+{
+	struct lightrec_registers *regs;
+	unsigned int i;
+
+	regs = lightrec_get_registers(lightrec_state);
+
+	printf("CYCLE 0x%08x PC 0x%08x", psxRegs.cycle, psxRegs.pc);
+
+	if (lightrec_very_debug)
+		printf(" RAM 0x%08x SCRATCH 0x%08x HW 0x%08x",
+				hash_calculate_ram(psxM, 0x200000),
+				hash_calculate_le(psxH, 0x400),
+				hash_calculate_le(psxH + 0x1000, 0x2000));
+
+	printf(" CP0 0x%08x CP2D 0x%08x CP2C 0x%08x INT 0x%04x INTCYCLE 0x%08x GPU 0x%08x",
+			hash_calculate(regs->cp0, sizeof(regs->cp0)),
+			hash_calculate(regs->cp2d, sizeof(regs->cp2d)),
+			hash_calculate(regs->cp2c, sizeof(regs->cp2c)),
+			psxRegs.interrupt,
+			hash_calculate(psxRegs.intCycle, sizeof(psxRegs.intCycle)),
+			LE32TOH(HW_GPU_STATUS));
+
+	if (lightrec_very_debug) {
+		for (i = 0; i < 32; i++)
+			printf(" CP2D%u 0x%08x", i, regs->cp2d[i]);
+		for (i = 0; i < 32; i++)
+			printf(" CP2C%u 0x%08x", i, regs->cp2c[i]);
+	}
+
+	if (lightrec_very_debug)
+		for (i = 0; i < 34; i++)
+			printf(" %s 0x%08x", mips_regs[i], regs->gpr[i]);
+	else
+		printf(" GPR 0x%08x",
+		       hash_calculate(regs->gpr, sizeof(regs->gpr)));
+	printf("\n");
+
+	fflush(stdout);
+}
+
 static void lightrec_plugin_sync_regs_to_pcsx(bool need_cp2);
 static void lightrec_plugin_sync_regs_from_pcsx(bool need_cp2);
 
 static void lightrec_plugin_execute_internal(bool block_only)
 {
 	struct lightrec_registers *regs;
 	u32 flags, cycles_pcsx;
+	u32 old_pc = psxRegs.pc;
 
 	regs = lightrec_get_registers(lightrec_state);
 	gen_interupt((psxCP0Regs *)regs->cp0);
@@ -522,6 +636,8 @@ static void lightrec_plugin_execute_internal(bool block_only)
 		if (flags & LIGHTREC_EXIT_SEGFAULT) {
 			fprintf(stderr, "Exiting at cycle 0x%08x\n",
 				psxRegs.cycle);
+			if (lightrec_debug)
+				print_for_big_ass_debugger();
 			exit(1);
 		}
 
@@ -542,6 +658,10 @@ static void lightrec_plugin_execute_internal(bool block_only)
 		}
 	}
 
+	if (lightrec_debug && psxRegs.cycle >= lightrec_begin_cycles && psxRegs.pc != old_pc) {
+		print_for_big_ass_debugger();
+	}
+
 	if ((regs->cp0[13] & regs->cp0[12] & 0x300) && (regs->cp0[12] & 0x1)) {
 		/* Handle software interrupts */
 		regs->cp0[13] &= ~0x7c;
@@ -552,7 +672,7 @@ static void lightrec_plugin_execute_internal(bool block_only)
 static void lightrec_plugin_execute(psxRegisters *regs)
 {
 	while (!regs->stop)
-		lightrec_plugin_execute_internal(false);
+		lightrec_plugin_execute_internal(lightrec_very_debug);
 }
 
 static void lightrec_plugin_execute_block(psxRegisters *regs,