GP-2677: Introduce TraceRmi (API only, experimental)

Ghidra/Test/IntegrationTest/src/test.slow/java/agent/gdb/rmi/AbstractGdbTraceRmiTest.java

@@ -0,0 +1,501 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package agent.gdb.rmi;
+
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerGUITest;
+import ghidra.app.plugin.core.debug.service.rmi.trace.*;
+import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
+import ghidra.app.services.TraceRmiService;
+import ghidra.dbg.target.TargetExecutionStateful.TargetExecutionState;
+import ghidra.dbg.testutil.DummyProc;
+import ghidra.framework.TestApplicationUtils;
+import ghidra.framework.main.ApplicationLevelOnlyPlugin;
+import ghidra.framework.model.DomainFile;
+import ghidra.framework.plugintool.Plugin;
+import ghidra.framework.plugintool.util.*;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.address.AddressRangeImpl;
+import ghidra.trace.model.breakpoint.TraceBreakpointKind;
+import ghidra.trace.model.breakpoint.TraceBreakpointKind.TraceBreakpointKindSet;
+import ghidra.trace.model.target.*;
+import ghidra.util.Msg;
+import ghidra.util.NumericUtilities;
+
+public abstract class AbstractGdbTraceRmiTest extends AbstractGhidraHeadedDebuggerGUITest {
+	// Connecting should be the first thing the script does, so use a tight timeout.
+	protected static final int CONNECT_TIMEOUT_MS = 3000;
+	protected static final int TIMEOUT_SECONDS = 300;
+	protected static final int QUIT_TIMEOUT_MS = 1000;
+	public static final String INSTRUMENT_STOPPED = """
+			ghidra trace tx-open "Fake" 'ghidra trace create-obj Inferiors[1]'
+			define do-set-stopped
+			  ghidra trace set-value Inferiors[1] _state '"STOPPED"'
+			end
+			define set-stopped
+			  ghidra trace tx-open Stopped do-set-stopped
+			end
+			python gdb.events.stop.connect(lambda e: gdb.execute("set-stopped"))""";
+	public static final String INSTRUMENT_RUNNING = """
+			ghidra trace tx-open "Fake" 'ghidra trace create-obj Inferiors[1]'
+			define do-set-running
+			  ghidra trace set-value Inferiors[1] _state '"RUNNING"'
+			end
+			define set-running
+			  ghidra trace tx-open Running do-set-running
+			end
+			python gdb.events.cont.connect(lambda e: gdb.execute("set-running"))""";
+
+	protected TraceRmiService traceRmi;
+	private Path gdbPath;
+	private Path outFile;
+	private Path errFile;
+
+	@BeforeClass
+	public static void setupPython() throws Throwable {
+		new ProcessBuilder("gradle", "Debugger-agent-gdb:installPyPackage")
+				.directory(TestApplicationUtils.getInstallationDirectory())
+				.inheritIO()
+				.start()
+				.waitFor();
+	}
+
+	protected Path getGdbPath() {
+		return Paths.get(DummyProc.which("gdb"));
+	}
+
+	@Before
+	public void setupTraceRmi() throws Throwable {
+		traceRmi = addPlugin(tool, TraceRmiPlugin.class);
+
+		gdbPath = getGdbPath();
+		outFile = Files.createTempFile("gdbout", null);
+		errFile = Files.createTempFile("gdberr", null);
+	}
+
+	protected void addAllDebuggerPlugins() throws PluginException {
+		PluginsConfiguration plugConf = new PluginsConfiguration() {
+			@Override
+			protected boolean accepts(Class<? extends Plugin> pluginClass) {
+				return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
+			}
+		};
+
+		for (PluginDescription pd : plugConf
+				.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
+			addPlugin(tool, pd.getPluginClass());
+		}
+	}
+
+	protected static String addrToStringForGdb(InetAddress address) {
+		if (address.isAnyLocalAddress()) {
+			return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
+		}
+		return address.getHostAddress();
+	}
+
+	protected static String sockToStringForGdb(SocketAddress address) {
+		if (address instanceof InetSocketAddress tcp) {
+			return addrToStringForGdb(tcp.getAddress()) + ":" + tcp.getPort();
+		}
+		throw new AssertionError("Unhandled address type " + address);
+	}
+
+	protected record GdbResult(boolean timedOut, int exitCode, String stdout, String stderr) {
+		protected String handle() {
+			if (!"".equals(stderr) | 0 != exitCode) {
+				throw new GdbError(exitCode, stdout, stderr);
+			}
+			return stdout;
+		}
+	}
+
+	protected record ExecInGdb(Process gdb, CompletableFuture<GdbResult> future) {
+	}
+
+	@SuppressWarnings("resource") // Do not close stdin 
+	protected ExecInGdb execInGdb(String script) throws IOException {
+		ProcessBuilder pb = new ProcessBuilder(gdbPath.toString());
+		// If commands come from file, GDB will quit after EOF.
+		Msg.info(this, "outFile: " + outFile);
+		Msg.info(this, "errFile: " + errFile);
+		pb.redirectInput(ProcessBuilder.Redirect.PIPE);
+		pb.redirectOutput(outFile.toFile());
+		pb.redirectError(errFile.toFile());
+		Process gdbProc = pb.start();
+		OutputStream stdin = gdbProc.getOutputStream();
+		stdin.write(script.getBytes());
+		stdin.flush();
+		return new ExecInGdb(gdbProc, CompletableFuture.supplyAsync(() -> {
+			try {
+				if (!gdbProc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+					Msg.error(this, "Timed out waiting for GDB");
+					gdbProc.destroyForcibly();
+					gdbProc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+					return new GdbResult(true, -1, Files.readString(outFile),
+						Files.readString(errFile));
+				}
+				Msg.info(this, "GDB exited with code " + gdbProc.exitValue());
+				return new GdbResult(false, gdbProc.exitValue(), Files.readString(outFile),
+					Files.readString(errFile));
+			}
+			catch (Exception e) {
+				return ExceptionUtils.rethrow(e);
+			}
+			finally {
+				gdbProc.destroyForcibly();
+			}
+		}));
+	}
+
+	protected static class GdbError extends RuntimeException {
+		public final int exitCode;
+		public final String stdout;
+		public final String stderr;
+
+		public GdbError(int exitCode, String stdout, String stderr) {
+			super("""
+					exitCode=%d:
+					----stdout----
+					%s
+					----stderr----
+					%s
+					""".formatted(exitCode, stdout, stderr));
+			this.exitCode = exitCode;
+			this.stdout = stdout;
+			this.stderr = stderr;
+		}
+	}
+
+	protected String runThrowError(String script) throws Exception {
+		CompletableFuture<GdbResult> result = execInGdb(script).future;
+		return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
+	}
+
+	protected record GdbAndHandler(ExecInGdb exec, TraceRmiHandler handler)
+			implements AutoCloseable {
+		protected RemoteMethod getMethod(String name) {
+			return Objects.requireNonNull(handler.getMethods().get(name));
+		}
+
+		public void execute(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			execute.invoke(Map.of("cmd", cmd));
+		}
+
+		public RemoteAsyncResult executeAsync(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			return execute.invokeAsync(Map.of("cmd", cmd));
+		}
+
+		public String executeCapture(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			return (String) execute.invoke(Map.of("cmd", cmd, "to_string", true));
+		}
+
+		@Override
+		public void close() throws Exception {
+			Msg.info(this, "Cleaning up gdb");
+			try {
+				try {
+					RemoteAsyncResult asyncQuit = executeAsync("quit");
+					try {
+						asyncQuit.get(QUIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+					}
+					catch (TimeoutException e) {
+						/**
+						 * This seems like a bug in gdb. AFAICT, it's a rehash or regression of
+						 * https://sourceware.org/bugzilla/show_bug.cgi?id=17247. If I attach to the
+						 * hung gdb, I get a similar stack trace, but with Python frames on the
+						 * stack. The workaround given in the comments works here, too. I hesitate
+						 * to point fingers, though, because I'm testing with a modern gdb-13.1
+						 * compiled from source on a rather un-modern distro.
+						 */
+						Msg.warn(this, "gdb hung on quit. Sending SIGCONT.");
+						Runtime.getRuntime().exec("kill -SIGCONT %d".formatted(exec.gdb.pid()));
+						asyncQuit.get(QUIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+					}
+				}
+				catch (TraceRmiError e) {
+					// expected
+				}
+				catch (ExecutionException e) {
+					if (!(e.getCause() instanceof TraceRmiError)) {
+						throw e;
+					}
+				}
+				GdbResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+				r.handle();
+				waitForPass(() -> assertTrue(handler.isClosed()));
+			}
+			finally {
+				exec.gdb.destroyForcibly();
+			}
+		}
+	}
+
+	protected GdbAndHandler startAndConnectGdb(Function<String, String> scriptSupplier)
+			throws Exception {
+		TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
+		ExecInGdb exec = execInGdb(scriptSupplier.apply(sockToStringForGdb(acceptor.getAddress())));
+		acceptor.setTimeout(CONNECT_TIMEOUT_MS);
+		try {
+			TraceRmiHandler handler = acceptor.accept();
+			return new GdbAndHandler(exec, handler);
+		}
+		catch (SocketTimeoutException e) {
+			exec.gdb.destroyForcibly();
+			exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
+			throw e;
+		}
+	}
+
+	protected GdbAndHandler startAndConnectGdb() throws Exception {
+		return startAndConnectGdb(addr -> """
+				set python print-stack full
+				python import ghidragdb
+				ghidra trace connect %s
+				""".formatted(addr));
+	}
+
+	@SuppressWarnings("resource")
+	protected String runThrowError(Function<String, String> scriptSupplier)
+			throws Exception {
+		GdbAndHandler conn = startAndConnectGdb(scriptSupplier);
+		GdbResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+		String stdout = r.handle();
+		waitForPass(() -> assertTrue(conn.handler.isClosed()));
+		return stdout;
+	}
+
+	protected void waitState(int infnum, Supplier<Long> snapSupplier, TargetExecutionState state) {
+		TraceObjectKeyPath infPath = TraceObjectKeyPath.parse("Inferiors").index(infnum);
+		TraceObject inf =
+			Objects.requireNonNull(tb.trace.getObjectManager().getObjectByCanonicalPath(infPath));
+		waitForPass(
+			() -> assertEquals(state.name(), tb.objValue(inf, snapSupplier.get(), "_state")));
+		waitTxDone();
+	}
+
+	protected void waitStopped() {
+		waitState(1, () -> 0L, TargetExecutionState.STOPPED);
+	}
+
+	protected void waitRunning() {
+		waitState(1, () -> 0L, TargetExecutionState.RUNNING);
+	}
+
+	protected String extractOutSection(String out, String head) {
+		return out.split(head)[1].split("---")[0].replace("(gdb)", "").trim();
+	}
+
+	record MemDump(long address, byte[] data) {
+	}
+
+	protected MemDump parseHexDump(String dump) throws IOException {
+		// First, get the address. Assume contiguous, so only need top line.
+		List<String> lines = List.of(dump.split("\n"));
+		List<String> toksLine0 = List.of(lines.get(0).split("\\s+"));
+		assertThat(toksLine0.get(0), startsWith("0x"));
+		long address = Long.decode(toksLine0.get(0));
+
+		ByteArrayOutputStream buf = new ByteArrayOutputStream();
+		for (String l : lines) {
+			List<String> parts = List.of(l.split(":"));
+			assertEquals(2, parts.size());
+			String hex = parts.get(1).replaceAll("\\s*0x", "");
+			byte[] lineData = NumericUtilities.convertStringToBytes(hex);
+			assertNotNull("Converted to null: " + hex, parts.get(1));
+			buf.write(lineData);
+		}
+		return new MemDump(address, buf.toByteArray());
+	}
+
+	record RegDump() {
+	}
+
+	protected RegDump parseRegDump(String dump) {
+		return new RegDump();
+	}
+
+	protected ManagedDomainObject openDomainObject(String path) throws Exception {
+		DomainFile df = env.getProject().getProjectData().getFile(path);
+		assertNotNull(df);
+		return new ManagedDomainObject(df, false, false, monitor);
+	}
+
+	protected ManagedDomainObject waitDomainObject(String path) throws Exception {
+		DomainFile df;
+		long start = System.currentTimeMillis();
+		while (true) {
+			df = env.getProject().getProjectData().getFile(path);
+			if (df != null) {
+				return new ManagedDomainObject(df, false, false, monitor);
+			}
+			Thread.sleep(1000);
+			if (System.currentTimeMillis() - start > 30000) {
+				throw new TimeoutException("30 seconds expired waiting for domain file");
+			}
+		}
+	}
+
+	protected void assertBreakLoc(TraceObjectValue locVal, String key, Address addr, int len,
+			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
+		assertEquals(key, locVal.getEntryKey());
+		TraceObject loc = locVal.getChild();
+		TraceObject spec = loc.getCanonicalParent(0).getParent();
+		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
+		assertEquals(TraceBreakpointKindSet.encode(kinds), spec.getValue(0, "_kinds").getValue());
+		assertEquals(expression, spec.getValue(0, "_expression").getValue());
+	}
+
+	protected void waitTxDone() {
+		waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
+	}
+
+	private record Cut(String head, int begin, int end) {
+		String parseCell(String line) {
+			int begin = Math.min(line.length(), this.begin);
+			int end = Math.min(line.length(), this.end);
+			/**
+			 * NOTE: Do not assert previous char is space.
+			 * 
+			 * When breakpoints table spells out locations, Address and What cells are indented and
+			 * no longer align with their column headers.
+			 */
+			return line.substring(begin, end).trim();
+		}
+	}
+
+	protected record Row(Map<String, String> cells) {
+		private static Row parse(List<Cut> cuts, String line) {
+			return new Row(
+				cuts.stream().collect(Collectors.toMap(Cut::head, c -> c.parseCell(line))));
+		}
+
+		public String getCell(String head) {
+			return cells.get(head);
+		}
+	}
+
+	protected record Tabular(List<String> headings, List<Row> rows) {
+		static final Pattern SPACES = Pattern.compile(" *");
+		static final Pattern WORDS = Pattern.compile("\\w+");
+
+		private static List<Cut> findCuts(String header) {
+			List<Cut> result = new ArrayList<>();
+			Matcher spaceMatcher = SPACES.matcher(header);
+			Matcher wordMatcher = WORDS.matcher(header);
+			int start = 0;
+			while (start < header.length()) {
+				if (!spaceMatcher.find(start)) {
+					throw new AssertionError();
+				}
+				start = spaceMatcher.end();
+				if (start >= header.length()) {
+					break;
+				}
+				if (!wordMatcher.find(start)) {
+					throw new AssertionError();
+				}
+				result.add(new Cut(wordMatcher.group(), wordMatcher.start(), wordMatcher.end()));
+				start = wordMatcher.end();
+			}
+			return result;
+		}
+
+		private static List<Cut> adjustCuts(List<Cut> cuts) {
+			List<Cut> result = new ArrayList<>();
+			for (int i = 0; i < cuts.size(); i++) {
+				Cut cut = cuts.get(i);
+				int j = i + 1;
+				int end = j < cuts.size() ? cuts.get(j).begin : Integer.MAX_VALUE;
+				result.add(new Cut(cut.head, cut.begin, end));
+			}
+			return result;
+		}
+
+		/**
+		 * Parse a table.
+		 * 
+		 * <p>
+		 * This is far from perfect, but good enough for making assertions in tests. For example, in
+		 * the breakpoints table, gdb may insert an extra informational line under a breakpoint row.
+		 * This line will get mangled and parsed as if it were an entry. However, it's "Num" cell
+		 * will be empty, so they will not likely interfere.
+		 * 
+		 * @param out the output in tabular form
+		 * @return the table object, more or less
+		 */
+		public static Tabular parse(String out) {
+			List<String> lines = List.of(out.split("\n"));
+			if (lines.isEmpty()) {
+				throw new AssertionError("Output is not tabular");
+			}
+			List<Cut> cuts = adjustCuts(findCuts(lines.get(0)));
+			return new Tabular(cuts.stream().map(Cut::head).toList(),
+				lines.stream().skip(1).map(l -> Row.parse(cuts, l)).toList());
+		}
+
+		public Row findRow(String head, String contents) {
+			return rows.stream()
+					.filter(r -> Objects.equals(contents, r.getCell(head)))
+					.findFirst()
+					.orElse(null);
+		}
+	}
+
+	public static void waitForPass(Runnable runnable, long timeoutMs, long retryDelayMs) {
+		long start = System.currentTimeMillis();
+		AssertionError lastError = null;
+		while (System.currentTimeMillis() - start < timeoutMs) {
+			try {
+				runnable.run();
+				return;
+			}
+			catch (AssertionError e) {
+				lastError = e;
+			}
+			try {
+				Thread.sleep(retryDelayMs);
+			}
+			catch (InterruptedException e) {
+				// Retry sooner, I guess.
+			}
+		}
+		if (lastError == null) {
+			throw new AssertionError("Timed out before first try?");
+		}
+		throw lastError;
+	}
+}

Ghidra/Test/IntegrationTest/src/test.slow/java/agent/gdb/rmi/GdbHooksTest.java

@@ -0,0 +1,425 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package agent.gdb.rmi;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import agent.gdb.model.GdbLinuxSpecimen;
+import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
+import ghidra.dbg.target.TargetExecutionStateful.TargetExecutionState;
+import ghidra.dbg.testutil.DummyProc;
+import ghidra.dbg.util.PathPattern;
+import ghidra.dbg.util.PathPredicates;
+import ghidra.program.model.address.AddressSpace;
+import ghidra.trace.database.ToyDBTraceBuilder;
+import ghidra.trace.model.Trace;
+import ghidra.trace.model.memory.TraceMemorySpace;
+import ghidra.trace.model.target.TraceObject;
+import ghidra.trace.model.time.TraceSnapshot;
+
+public class GdbHooksTest extends AbstractGdbTraceRmiTest {
+	private static final long RUN_TIMEOUT_MS = 20000;
+	private static final long RETRY_MS = 500;
+
+	record GdbAndTrace(GdbAndHandler conn, ManagedDomainObject mdo) implements AutoCloseable {
+		public void execute(String cmd) {
+			conn.execute(cmd);
+		}
+
+		public String executeCapture(String cmd) {
+			return conn.executeCapture(cmd);
+		}
+
+		@Override
+		public void close() throws Exception {
+			conn.close();
+			mdo.close();
+		}
+	}
+
+	@SuppressWarnings("resource")
+	protected GdbAndTrace startAndSyncGdb() throws Exception {
+		GdbAndHandler conn = startAndConnectGdb();
+		try {
+			// TODO: Why does using 'set arch' cause a hang at quit?
+			conn.execute("""
+					set ghidra-language x86:LE:64:default
+					ghidra trace start
+					ghidra trace sync-enable""");
+			ManagedDomainObject mdo = waitDomainObject("/New Traces/gdb/noname");
+			tb = new ToyDBTraceBuilder((Trace) mdo.get());
+			return new GdbAndTrace(conn, mdo);
+		}
+		catch (Exception e) {
+			conn.close();
+			throw e;
+		}
+	}
+
+	@Test
+	public void testOnNewInferior() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("add-inferior");
+			waitForPass(() -> assertEquals(2, tb.objValues(0, "Inferiors[]").size()));
+		}
+	}
+
+	protected String getIndex(TraceObject object, String pattern) {
+		if (object == null) {
+			return null;
+		}
+		PathPattern pat = PathPredicates.parse(pattern).getSingletonPattern();
+		if (pat.countWildcards() != 1) {
+			throw new IllegalArgumentException("Exactly one wildcard required");
+		}
+		List<String> path = object.getCanonicalPath().getKeyList();
+		if (path.size() < pat.asPath().size()) {
+			return null;
+		}
+		List<String> matched = pat.matchKeys(path.subList(0, pat.asPath().size()));
+		if (matched == null) {
+			return null;
+		}
+		return matched.get(0);
+	}
+
+	protected String inferiorIndex(TraceObject object) {
+		return getIndex(object, "Inferiors[]");
+	}
+
+	@Test
+	public void testOnInferiorSelected() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			traceManager.openTrace(tb.trace);
+			// Both inferiors must have sync enabled
+			conn.execute("""
+					add-inferior
+					inferior 2
+					ghidra trace sync-enable""");
+
+			conn.execute("inferior 1");
+			waitForPass(() -> assertEquals("1", inferiorIndex(traceManager.getCurrentObject())));
+
+			conn.execute("inferior 2");
+			waitForPass(() -> assertEquals("2", inferiorIndex(traceManager.getCurrentObject())));
+
+			conn.execute("inferior 1");
+			waitForPass(() -> assertEquals("1", inferiorIndex(traceManager.getCurrentObject())));
+		}
+	}
+
+	@Test
+	public void testOnInferiorDeleted() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("add-inferior");
+			waitForPass(() -> assertEquals(2, tb.objValues(0, "Inferiors[]").size()));
+
+			conn.execute("remove-inferior 2");
+			waitForPass(() -> assertEquals(1, tb.objValues(0, "Inferiors[]").size()));
+		}
+	}
+
+	protected long lastSnap(GdbAndTrace conn) {
+		return conn.conn.handler().getLastSnapshot(tb.trace);
+	}
+
+	@Test
+	public void testOnNewThread() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file %s
+					break work
+					start""".formatted(GdbLinuxSpecimen.CLONE_EXIT.getCommandLine()));
+			waitForPass(() -> {
+				TraceObject inf = tb.obj("Inferiors[1]");
+				assertNotNull(inf);
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+			waitForPass(() -> assertEquals(1,
+				tb.objValues(lastSnap(conn), "Inferiors[1].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("continue");
+			waitForPass(() -> assertEquals(2,
+				tb.objValues(lastSnap(conn), "Inferiors[1].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	protected String threadIndex(TraceObject object) {
+		return getIndex(object, "Inferiors[1].Threads[]");
+	}
+
+	@Test
+	public void testOnThreadSelected() throws Exception {
+		String cloneExit = DummyProc.which("expCloneExit");
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			traceManager.openTrace(tb.trace);
+
+			conn.execute("""
+					file %s
+					break work
+					run""".formatted(cloneExit));
+			waitForPass(() -> {
+				TraceObject inf = tb.obj("Inferiors[1]");
+				assertNotNull(inf);
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+			waitForPass(() -> assertEquals(2,
+				tb.objValues(lastSnap(conn), "Inferiors[1].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			// Now the real test
+			conn.execute("thread 1");
+			waitForPass(() -> assertEquals("1", threadIndex(traceManager.getCurrentObject())));
+
+			conn.execute("thread 2");
+			waitForPass(() -> assertEquals("2", threadIndex(traceManager.getCurrentObject())));
+
+			conn.execute("thread 1");
+			waitForPass(() -> assertEquals("1", threadIndex(traceManager.getCurrentObject())));
+		}
+	}
+
+	protected String frameIndex(TraceObject object) {
+		return getIndex(object, "Inferiors[1].Threads[1].Stack[]");
+	}
+
+	@Test
+	public void testOnFrameSelected() throws Exception {
+		String stack = DummyProc.which("expStack");
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			traceManager.openTrace(tb.trace);
+
+			conn.execute("""
+					file %s
+					break break_here
+					run""".formatted(stack));
+			waitForPass(() -> assertThat(
+				tb.objValues(lastSnap(conn), "Inferiors[1].Threads[1].Stack[]").size(),
+				greaterThan(2)),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("frame 1");
+			waitForPass(() -> assertEquals("1", frameIndex(traceManager.getCurrentObject())),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("frame 0");
+			waitForPass(() -> assertEquals("0", frameIndex(traceManager.getCurrentObject())),
+				RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	@Ignore
+	public void testOnSyscallMemory() throws Exception {
+		// TODO: Need a specimen
+		// FWIW, I've already seen this getting exercised in other tests.
+	}
+
+	@Test
+	public void testOnMemoryChanged() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file bash
+					start""");
+
+			long address = Long.decode(conn.executeCapture("print/x &main").split("\\s+")[2]);
+			conn.execute("set *((char*) &main) = 0x7f");
+			waitForPass(() -> {
+				ByteBuffer buf = ByteBuffer.allocate(1);
+				tb.trace.getMemoryManager().getBytes(lastSnap(conn), tb.addr(address), buf);
+				assertEquals(0x7f, buf.get(0));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnRegisterChanged() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file bash
+					start""");
+
+			TraceObject thread = waitForValue(() -> tb.obj("Inferiors[1].Threads[1]"));
+			waitForPass(
+				() -> assertEquals("STOPPED", tb.objValue(thread, lastSnap(conn), "_state")),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("set $rax = 0x1234");
+			AddressSpace space = tb.trace.getBaseAddressFactory()
+					.getAddressSpace("Inferiors[1].Threads[1].Stack[0].Registers");
+			TraceMemorySpace regs = tb.trace.getMemoryManager().getMemorySpace(space, false);
+			waitForPass(() -> assertEquals("1234",
+				regs.getValue(lastSnap(conn), tb.reg("RAX")).getUnsignedValue().toString(16)));
+		}
+	}
+
+	@Test
+	public void testOnCont() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file bash
+					run""");
+
+			TraceObject inf = waitForValue(() -> tb.obj("Inferiors[1]"));
+			TraceObject thread = waitForValue(() -> tb.obj("Inferiors[1].Threads[1]"));
+			waitForPass(() -> {
+				assertEquals("RUNNING", tb.objValue(inf, lastSnap(conn), "_state"));
+				assertEquals("RUNNING", tb.objValue(thread, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnStop() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file bash
+					start""");
+
+			TraceObject inf = waitForValue(() -> tb.obj("Inferiors[1]"));
+			TraceObject thread = waitForValue(() -> tb.obj("Inferiors[1].Threads[1]"));
+			waitForPass(() -> {
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+				assertEquals("STOPPED", tb.objValue(thread, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnExited() throws Exception {
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file bash
+					set args -c "exit 1"
+					run""");
+
+			waitForPass(() -> {
+				TraceSnapshot snapshot =
+					tb.trace.getTimeManager().getSnapshot(lastSnap(conn), false);
+				assertNotNull(snapshot);
+				assertEquals("Exited with code 1", snapshot.getDescription());
+
+				TraceObject inf1 = tb.obj("Inferiors[1]");
+				assertNotNull(inf1);
+				Object val = tb.objValue(inf1, lastSnap(conn), "_exit_code");
+				assertThat(val, instanceOf(Number.class));
+				assertEquals(1, ((Number) val).longValue());
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	/**
+	 * Test on_clear_objfiles, on_new_objfile, on_free_objfile.
+	 * 
+	 * <p>
+	 * Technically, this probably doesn't hit on_free_objfile, but all three just call
+	 * modules_changed, so I'm not concerned.
+	 */
+	@Test
+	public void testOnEventsObjfiles() throws Exception {
+		String print = DummyProc.which("expPrint");
+		String modPrint = "Inferiors[1].Modules[%s]".formatted(print);
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("""
+					file %s
+					start""".formatted(print));
+			waitForPass(() -> assertEquals(1, tb.objValues(lastSnap(conn), modPrint).size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("continue");
+			waitState(1, () -> lastSnap(conn), TargetExecutionState.TERMINATED);
+			/**
+			 * Termination does not clear objfiles. Not until we run a new target.
+			 */
+			conn.execute("""
+					file bash
+					set args -c "exit 1"
+					run""");
+			waitForPass(() -> assertEquals(0, tb.objValues(lastSnap(conn), modPrint).size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnBreakpointCreated() throws Exception {
+		String print = DummyProc.which("expPrint");
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("file " + print);
+			assertEquals(0, tb.objValues(lastSnap(conn), "Breakpoints[]").size());
+
+			conn.execute("break main");
+			waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+		}
+	}
+
+	@Test
+	public void testOnBreakpointModified() throws Exception {
+		String print = DummyProc.which("expPrint");
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("file " + print);
+			assertEquals(0, tb.objValues(lastSnap(conn), "Breakpoints[]").size());
+
+			conn.execute("break main");
+			TraceObject brk = waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+			assertEquals(null, tb.objValue(brk, lastSnap(conn), "Commands"));
+
+			conn.execute("""
+					commands %s
+					  echo test
+					end""".formatted(brk.getCanonicalPath().index()));
+			waitForPass(
+				() -> assertEquals("echo test\n", tb.objValue(brk, lastSnap(conn), "Commands")));
+		}
+	}
+
+	@Test
+	public void testOnBreakpointDeleted() throws Exception {
+		String print = DummyProc.which("expPrint");
+		try (GdbAndTrace conn = startAndSyncGdb()) {
+			conn.execute("file " + print);
+			assertEquals(0, tb.objValues(lastSnap(conn), "Breakpoints[]").size());
+
+			conn.execute("break main");
+			TraceObject brk = waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+
+			conn.execute("delete %s".formatted(brk.getCanonicalPath().index()));
+			waitForPass(
+				() -> assertEquals(0, tb.objValues(lastSnap(conn), "Breakpoints[]").size()));
+		}
+	}
+}

Ghidra/Test/IntegrationTest/src/test.slow/java/agent/lldb/rmi/AbstractLldbTraceRmiTest.java

@@ -0,0 +1,509 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package agent.lldb.rmi;
+
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerGUITest;
+import ghidra.app.plugin.core.debug.service.rmi.trace.RemoteAsyncResult;
+import ghidra.app.plugin.core.debug.service.rmi.trace.RemoteMethod;
+import ghidra.app.plugin.core.debug.service.rmi.trace.TraceRmiAcceptor;
+import ghidra.app.plugin.core.debug.service.rmi.trace.TraceRmiHandler;
+import ghidra.app.plugin.core.debug.service.rmi.trace.TraceRmiPlugin;
+import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
+import ghidra.app.services.TraceRmiService;
+import ghidra.dbg.testutil.DummyProc;
+import ghidra.framework.TestApplicationUtils;
+import ghidra.framework.main.ApplicationLevelOnlyPlugin;
+import ghidra.framework.model.DomainFile;
+import ghidra.framework.plugintool.Plugin;
+import ghidra.framework.plugintool.util.*;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.address.AddressRangeImpl;
+import ghidra.trace.model.Lifespan;
+import ghidra.trace.model.breakpoint.TraceBreakpointKind;
+import ghidra.trace.model.breakpoint.TraceBreakpointKind.TraceBreakpointKindSet;
+import ghidra.trace.model.target.TraceObject;
+import ghidra.trace.model.target.TraceObjectValue;
+import ghidra.util.Msg;
+import ghidra.util.NumericUtilities;
+
+public abstract class AbstractLldbTraceRmiTest extends AbstractGhidraHeadedDebuggerGUITest {
+	// Connecting should be the first thing the script does, so use a tight timeout.
+	protected static final int CONNECT_TIMEOUT_MS = 3000;
+	protected static final int TIMEOUT_SECONDS = 300;
+	protected static final int QUIT_TIMEOUT_MS = 1000;
+	public static final String INSTRUMENT_STOPPED =
+		"""
+				ghidra_trace_txopen "Fake" 'ghidra_trace_create_obj Processes[1]'
+				define do-set-stopped
+				  ghidra_trace_set_value Processes[1] _state '"STOPPED"'
+				end
+				define set-stopped
+				  ghidra_trace_txopen Stopped do-set-stopped
+				end
+				         #lldb.debugger.HandleCommand('target stop-hook add -P ghidralldb.hooks.StopHook')
+				#python lldb.events.stop.connect(lambda e: lldb.execute("set-stopped"))""";
+	public static final String INSTRUMENT_RUNNING =
+		"""
+				ghidra_trace_txopen "Fake" 'ghidra_trace_create_obj Processes[1]'
+				define do-set-running
+				  ghidra_trace_set_value Processes[1] _state '"RUNNING"'
+				end
+				define set-running
+				  ghidra_trace_txopen Running do-set-running
+				end
+				         #lldb.debugger.HandleCommand('target stop-hook add -P ghidralldb.hooks.StopHook')
+				#python lldb.events.cont.connect(lambda e: lldb.execute("set-running"))""";
+
+	protected TraceRmiService traceRmi;
+	private Path lldbPath;
+	private Path outFile;
+	private Path errFile;
+
+	@BeforeClass
+	public static void setupPython() throws Throwable {
+		new ProcessBuilder("gradle", "Debugger-agent-lldb:installPyPackage")
+				.directory(TestApplicationUtils.getInstallationDirectory())
+				.inheritIO()
+				.start()
+				.waitFor();
+	}
+
+	@Before
+	public void setupTraceRmi() throws Throwable {
+		traceRmi = addPlugin(tool, TraceRmiPlugin.class);
+
+		lldbPath = Paths.get(DummyProc.which("lldb"));
+		outFile = Files.createTempFile("lldbout", null);
+		errFile = Files.createTempFile("lldberr", null);
+	}
+
+	protected void addAllDebuggerPlugins() throws PluginException {
+		PluginsConfiguration plugConf = new PluginsConfiguration() {
+			@Override
+			protected boolean accepts(Class<? extends Plugin> pluginClass) {
+				return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
+			}
+		};
+
+		for (PluginDescription pd : plugConf
+				.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
+			addPlugin(tool, pd.getPluginClass());
+		}
+	}
+
+	protected static String addrToStringForLldb(InetAddress address) {
+		if (address.isAnyLocalAddress()) {
+			return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
+		}
+		return address.getHostAddress();
+	}
+
+	protected static String sockToStringForLldb(SocketAddress address) {
+		if (address instanceof InetSocketAddress tcp) {
+			return addrToStringForLldb(tcp.getAddress()) + ":" + tcp.getPort();
+		}
+		throw new AssertionError("Unhandled address type " + address);
+	}
+
+	protected record LldbResult(boolean timedOut, int exitCode, String stdout, String stderr) {
+		protected String handle() {
+			if (!"".equals(stderr) || (0 != exitCode && 143 != exitCode)) {
+				throw new LldbError(exitCode, stdout, stderr);
+			}
+			return stdout;
+		}
+	}
+
+	protected record ExecInLldb(Process lldb, CompletableFuture<LldbResult> future) {
+	}
+
+	@SuppressWarnings("resource") // Do not close stdin 
+	protected ExecInLldb execInLldb(String script) throws IOException {
+		ProcessBuilder pb = new ProcessBuilder(lldbPath.toString());
+		// If commands come from file, LLDB will quit after EOF.
+		Msg.info(this, "outFile: " + outFile);
+		Msg.info(this, "errFile: " + errFile);
+		pb.redirectInput(ProcessBuilder.Redirect.PIPE);
+		pb.redirectOutput(outFile.toFile());
+		pb.redirectError(errFile.toFile());
+		Process lldbProc = pb.start();
+		OutputStream stdin = lldbProc.getOutputStream();
+		stdin.write(script.getBytes());
+		stdin.flush();
+		return new ExecInLldb(lldbProc, CompletableFuture.supplyAsync(() -> {
+			try {
+				if (!lldbProc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+					Msg.error(this, "Timed out waiting for LLDB");
+					lldbProc.destroyForcibly();
+					lldbProc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+					return new LldbResult(true, -1, Files.readString(outFile),
+						Files.readString(errFile));
+				}
+				Msg.info(this, "LLDB exited with code " + lldbProc.exitValue());
+				return new LldbResult(false, lldbProc.exitValue(), Files.readString(outFile),
+					Files.readString(errFile));
+			}
+			catch (Exception e) {
+				return ExceptionUtils.rethrow(e);
+			}
+			finally {
+				lldbProc.destroyForcibly();
+			}
+		}));
+	}
+
+	public static class LldbError extends RuntimeException {
+		public final int exitCode;
+		public final String stdout;
+		public final String stderr;
+
+		public LldbError(int exitCode, String stdout, String stderr) {
+			super("""
+					exitCode=%d:
+					----stdout----
+					%s
+					----stderr----
+					%s
+					""".formatted(exitCode, stdout, stderr));
+			this.exitCode = exitCode;
+			this.stdout = stdout;
+			this.stderr = stderr;
+		}
+	}
+
+	protected String runThrowError(String script) throws Exception {
+		CompletableFuture<LldbResult> result = execInLldb(script).future;
+		return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
+	}
+
+	protected record LldbAndHandler(ExecInLldb exec, TraceRmiHandler handler)
+			implements AutoCloseable {
+		protected RemoteMethod getMethod(String name) {
+			return Objects.requireNonNull(handler.getMethods().get(name));
+		}
+
+		public void execute(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			execute.invoke(Map.of("cmd", cmd));
+		}
+
+		public RemoteAsyncResult executeAsync(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			return execute.invokeAsync(Map.of("cmd", cmd));
+		}
+
+		public String executeCapture(String cmd) {
+			RemoteMethod execute = getMethod("execute");
+			return (String) execute.invoke(Map.of("cmd", cmd, "to_string", true));
+		}
+
+		@Override
+		public void close() throws Exception {
+			Msg.info(this, "Cleaning up lldb");
+			exec.lldb().destroy();
+			try {
+				LldbResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+				r.handle();
+				waitForPass(() -> assertTrue(handler.isClosed()));
+			}
+			finally {
+				exec.lldb.destroyForcibly();
+			}
+		}
+	}
+
+	protected LldbAndHandler startAndConnectLldb(Function<String, String> scriptSupplier)
+			throws Exception {
+		TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
+		ExecInLldb exec =
+			execInLldb(scriptSupplier.apply(sockToStringForLldb(acceptor.getAddress())));
+		acceptor.setTimeout(CONNECT_TIMEOUT_MS);
+		try {
+			TraceRmiHandler handler = acceptor.accept();
+			return new LldbAndHandler(exec, handler);
+		}
+		catch (SocketTimeoutException e) {
+			exec.lldb.destroyForcibly();
+			exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
+			throw e;
+		}
+	}
+
+	protected LldbAndHandler startAndConnectLldb() throws Exception {
+		return startAndConnectLldb(addr -> """
+				script import ghidralldb
+				ghidra_trace_connect %s
+				""".formatted(addr));
+	}
+
+	@SuppressWarnings("resource")
+	protected String runThrowError(Function<String, String> scriptSupplier)
+			throws Exception {
+		LldbAndHandler conn = startAndConnectLldb(scriptSupplier);
+		LldbResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
+		String stdout = r.handle();
+		waitForPass(() -> assertTrue(conn.handler.isClosed()));
+		return stdout;
+	}
+
+	protected void waitStopped() {
+		TraceObject proc = Objects.requireNonNull(tb.objAny("Processes[]", Lifespan.at(0)));
+		waitForPass(() -> assertEquals("STOPPED", tb.objValue(proc, 0, "_state")));
+		waitTxDone();
+	}
+
+	protected void waitRunning() {
+		TraceObject proc = Objects.requireNonNull(tb.objAny("Processes[]", Lifespan.at(0)));
+		waitForPass(() -> assertEquals("RUNNING", tb.objValue(proc, 0, "_state")));
+		waitTxDone();
+	}
+
+	protected String extractOutSection(String out, String head) {
+		String[] split = out.split("\n");
+		String xout = "";
+		for (String s : split) {
+			if (!s.startsWith("(lldb)") && !s.equals("")) {
+				xout += s + "\n";
+			}
+		}
+		return xout.split(head)[1].split("---")[0].replace("(lldb)", "").trim();
+	}
+
+	record MemDump(long address, byte[] data) {
+	}
+
+	protected MemDump parseHexDump(String dump) throws IOException {
+		// First, get the address. Assume contiguous, so only need top line.
+		List<String> lines = List.of(dump.split("\n"));
+		List<String> toksLine0 = List.of(lines.get(0).split("\\s+"));
+		assertThat(toksLine0.get(0), startsWith("0x"));
+		String addrstr = toksLine0.get(0);
+		if (addrstr.contains(":")) {
+			addrstr = addrstr.substring(0, addrstr.indexOf(":"));
+		}
+		long address = Long.decode(addrstr);
+
+		ByteArrayOutputStream buf = new ByteArrayOutputStream();
+		for (String l : lines) {
+			List<String> parts = List.of(l.split(":"));
+			assertEquals(2, parts.size());
+			String hex = parts.get(1).replaceAll("\\s*0x", "");
+			byte[] lineData = NumericUtilities.convertStringToBytes(hex);
+			assertNotNull("Converted to null: " + hex, parts.get(1));
+			buf.write(lineData);
+		}
+		return new MemDump(address, buf.toByteArray());
+	}
+
+	record RegDump() {
+	}
+
+	protected RegDump parseRegDump(String dump) {
+		return new RegDump();
+	}
+
+	protected ManagedDomainObject openDomainObject(String path) throws Exception {
+		DomainFile df = env.getProject().getProjectData().getFile(path);
+		assertNotNull(df);
+		return new ManagedDomainObject(df, false, false, monitor);
+	}
+
+	protected ManagedDomainObject waitDomainObject(String path) throws Exception {
+		DomainFile df;
+		long start = System.currentTimeMillis();
+		while (true) {
+			df = env.getProject().getProjectData().getFile(path);
+			if (df != null) {
+				return new ManagedDomainObject(df, false, false, monitor);
+			}
+			Thread.sleep(1000);
+			if (System.currentTimeMillis() - start > 30000) {
+				throw new TimeoutException("30 seconds expired waiting for domain file");
+			}
+		}
+	}
+
+	protected void assertBreakLoc(TraceObjectValue locVal, String key, Address addr, int len,
+			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
+		assertEquals(key, locVal.getEntryKey());
+		TraceObject loc = locVal.getChild();
+		TraceObject spec = loc.getCanonicalParent(0).getParent();
+		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
+		assertEquals(TraceBreakpointKindSet.encode(kinds), spec.getValue(0, "_kinds").getValue());
+		assertTrue(spec.getValue(0, "_expression").getValue().toString().contains(expression));
+	}
+
+	protected void assertWatchLoc(TraceObjectValue locVal, String key, Address addr, int len,
+			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
+		assertEquals(key, locVal.getEntryKey());
+		TraceObject loc = locVal.getChild();
+		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
+		assertEquals(TraceBreakpointKindSet.encode(kinds), loc.getValue(0, "_kinds").getValue());
+	}
+
+	protected void waitTxDone() {
+		waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
+	}
+
+	private record Cut(String head, int begin, int end) {
+		String parseCell(String line) {
+			int begin = Math.min(line.length(), this.begin);
+			int end = Math.min(line.length(), this.end);
+			/**
+			 * NOTE: Do not assert previous char is space.
+			 * 
+			 * When breakpoints table spells out locations, Address and What cells are indented and
+			 * no longer align with their column headers.
+			 */
+			return line.substring(begin, end).trim();
+		}
+	}
+
+	protected record Row(Map<String, String> cells) {
+		private static Row parse(List<Cut> cuts, String line) {
+			return new Row(
+				cuts.stream().collect(Collectors.toMap(Cut::head, c -> c.parseCell(line))));
+		}
+
+		public String getCell(String head) {
+			return cells.get(head);
+		}
+	}
+
+	protected record Tabular(List<String> headings, List<Row> rows) {
+		static final Pattern SPACES = Pattern.compile(" *");
+		static final Pattern WORDS = Pattern.compile("\\w+");
+
+		private static List<Cut> findCuts(String header) {
+			List<Cut> result = new ArrayList<>();
+			Matcher spaceMatcher = SPACES.matcher(header);
+			Matcher wordMatcher = WORDS.matcher(header);
+			int start = 0;
+			while (start < header.length()) {
+				if (!spaceMatcher.find(start)) {
+					throw new AssertionError();
+				}
+				start = spaceMatcher.end();
+				if (start >= header.length()) {
+					break;
+				}
+				if (!wordMatcher.find(start)) {
+					throw new AssertionError();
+				}
+				result.add(new Cut(wordMatcher.group(), wordMatcher.start(), wordMatcher.end()));
+				start = wordMatcher.end();
+			}
+			return result;
+		}
+
+		private static List<Cut> adjustCuts(List<Cut> cuts) {
+			List<Cut> result = new ArrayList<>();
+			for (int i = 0; i < cuts.size(); i++) {
+				Cut cut = cuts.get(i);
+				int j = i + 1;
+				int end = j < cuts.size() ? cuts.get(j).begin : Integer.MAX_VALUE;
+				result.add(new Cut(cut.head, cut.begin, end));
+			}
+			return result;
+		}
+
+		/**
+		 * Parse a table.
+		 * 
+		 * <p>
+		 * This is far from perfect, but good enough for making assertions in tests. For example, in
+		 * the breakpoints table, lldb may insert an extra informational line under a breakpoint
+		 * row. This line will get mangled and parsed as if it were an entry. However, it's "Num"
+		 * cell will be empty, so they will not likely interfere.
+		 * 
+		 * @param out the output in tabular form
+		 * @return the table object, more or less
+		 */
+		public static Tabular parse(String out) {
+			List<String> lines = List.of(out.split("\n"));
+			if (lines.isEmpty()) {
+				throw new AssertionError("Output is not tabular");
+			}
+			List<Cut> cuts = adjustCuts(findCuts(lines.get(0)));
+			return new Tabular(cuts.stream().map(Cut::head).toList(),
+				lines.stream().skip(1).map(l -> Row.parse(cuts, l)).toList());
+		}
+
+		public Row findRow(String head, String contents) {
+			return rows.stream()
+					.filter(r -> Objects.equals(contents, r.getCell(head)))
+					.findFirst()
+					.orElse(null);
+		}
+	}
+
+	public static void waitForPass(Runnable runnable, long timeoutMs, long retryDelayMs) {
+		long start = System.currentTimeMillis();
+		AssertionError lastError = null;
+		while (System.currentTimeMillis() - start < timeoutMs) {
+			try {
+				runnable.run();
+				return;
+			}
+			catch (AssertionError e) {
+				lastError = e;
+			}
+			try {
+				Thread.sleep(retryDelayMs);
+			}
+			catch (InterruptedException e) {
+				// Retry sooner, I guess.
+			}
+		}
+		if (lastError == null) {
+			throw new AssertionError("Timed out before first try?");
+		}
+		throw lastError;
+	}
+}

Ghidra/Test/IntegrationTest/src/test.slow/java/agent/lldb/rmi/LldbHooksTest.java

@@ -0,0 +1,407 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package agent.lldb.rmi;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Objects;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import agent.gdb.model.GdbLinuxSpecimen;
+import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
+import ghidra.dbg.util.PathPattern;
+import ghidra.dbg.util.PathPredicates;
+import ghidra.program.model.address.AddressSpace;
+import ghidra.trace.database.ToyDBTraceBuilder;
+import ghidra.trace.model.Lifespan;
+import ghidra.trace.model.Trace;
+import ghidra.trace.model.memory.TraceMemorySpace;
+import ghidra.trace.model.target.TraceObject;
+import ghidra.trace.model.time.TraceSnapshot;
+
+public class LldbHooksTest extends AbstractLldbTraceRmiTest {
+	private static final long RUN_TIMEOUT_MS = 20000;
+	private static final long RETRY_MS = 500;
+
+	record LldbAndTrace(LldbAndHandler conn, ManagedDomainObject mdo) implements AutoCloseable {
+		public void execute(String cmd) {
+			conn.execute(cmd);
+		}
+
+		public String executeCapture(String cmd) {
+			return conn.executeCapture(cmd);
+		}
+
+		@Override
+		public void close() throws Exception {
+			conn.close();
+			mdo.close();
+		}
+	}
+
+	@SuppressWarnings("resource")
+	protected LldbAndTrace startAndSyncLldb() throws Exception {
+		LldbAndHandler conn = startAndConnectLldb();
+		try {
+			// TODO: Why does using 'set arch' cause a hang at quit?
+			conn.execute("ghidralldb.util.set_convenience_variable('ghidra-language', 'x86:LE:64:default')");
+			conn.execute("ghidra_trace_start");
+			ManagedDomainObject mdo = waitDomainObject("/New Traces/lldb/noname");
+			tb = new ToyDBTraceBuilder((Trace) mdo.get());
+			return new LldbAndTrace(conn, mdo);
+		}
+		catch (Exception e) {
+			conn.close();
+			throw e;
+		}
+	}
+
+	protected long lastSnap(LldbAndTrace conn) {
+		return conn.conn.handler().getLastSnapshot(tb.trace);
+	}
+
+	// TODO: This passes if you single-step through it but fails on some transactional stuff if run
+	//@Test
+	public void testOnNewThread() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+
+			start(conn, "%s".formatted(GdbLinuxSpecimen.CLONE_EXIT.getCommandLine()));
+			conn.execute("break set -n work");
+			waitForPass(() -> {
+				TraceObject proc = tb.objAny("Processes[]");
+				assertNotNull(proc);
+				assertEquals("STOPPED", tb.objValue(proc, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+			
+			txPut(conn, "threads");
+			waitForPass(() -> assertEquals(1,
+				tb.objValues(lastSnap(conn), "Processes[].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("continue");
+			waitStopped();
+			txPut(conn, "threads");
+			waitForPass(() -> assertEquals(2,
+				tb.objValues(lastSnap(conn), "Processes[].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	// TODO: This passes if you single-step through it but fails on some transactional stuff if run
+	//@Test
+	public void testOnThreadSelected() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			traceManager.openTrace(tb.trace);
+
+			start(conn, "%s".formatted(GdbLinuxSpecimen.CLONE_EXIT.getCommandLine()));
+			conn.execute("break set -n work");
+			
+			waitForPass(() -> {
+				TraceObject inf = tb.objAny("Processes[]");
+				assertNotNull(inf);
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+			txPut(conn, "threads");
+			waitForPass(() -> assertEquals(1,
+				tb.objValues(lastSnap(conn), "Processes[].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("continue");
+			waitStopped();
+			waitForPass(() -> {
+				TraceObject inf = tb.objAny("Processes[]");
+				assertNotNull(inf);
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+			
+			waitForPass(() -> assertEquals(2,
+				tb.objValues(lastSnap(conn), "Processes[].Threads[]").size()),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			// Now the real test
+			conn.execute("thread select 1");
+			conn.execute("frame select 0");
+			waitForPass(() -> {
+				String ti0 = conn.executeCapture("thread info");
+				assertTrue(ti0.contains("#1"));
+			    String threadIndex = threadIndex(traceManager.getCurrentObject());
+				assertTrue(ti0.contains(threadIndex));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("thread select 2");
+			conn.execute("frame select 0");
+			waitForPass(() -> {
+				String ti0 = conn.executeCapture("thread info");
+				assertTrue(ti0.contains("#2"));
+			    String threadIndex = threadIndex(traceManager.getCurrentObject());
+				assertTrue(ti0.contains(threadIndex));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("thread select 1");
+			conn.execute("frame select 0");
+			waitForPass(() -> {
+				String ti0 = conn.executeCapture("thread info");
+				assertTrue(ti0.contains("#1"));
+			    String threadIndex = threadIndex(traceManager.getCurrentObject());
+				assertTrue(ti0.contains(threadIndex));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	protected String getIndex(TraceObject object, String pattern, int n) {
+		if (object == null) {
+			return null;
+		}
+		PathPattern pat = PathPredicates.parse(pattern).getSingletonPattern();
+//		if (pat.countWildcards() != 1) {
+//			throw new IllegalArgumentException("Exactly one wildcard required");
+//		}
+		List<String> path = object.getCanonicalPath().getKeyList();
+		if (path.size() < pat.asPath().size()) {
+			return null;
+		}
+		List<String> matched = pat.matchKeys(path.subList(0, pat.asPath().size()));
+		if (matched == null) {
+			return null;
+		}
+		if (matched.size() <= n) {
+			return null;
+		}
+		return matched.get(n);
+	}
+
+	protected String threadIndex(TraceObject object) {
+		return getIndex(object, "Processes[].Threads[]", 1);
+	}
+
+	protected String frameIndex(TraceObject object) {
+		return getIndex(object, "Processes[].Threads[].Stack[]", 2);
+	}
+
+	@Test
+	public void testOnFrameSelected() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			traceManager.openTrace(tb.trace);
+
+			start(conn, "bash");
+			conn.execute("breakpoint set -n read");
+			conn.execute("cont");
+			
+			waitStopped();
+			waitForPass(() -> assertThat(
+				tb.objValues(lastSnap(conn), "Processes[].Threads[].Stack[]").size(),
+				greaterThan(2)),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("frame select 1");
+			waitForPass(() -> assertEquals("1", frameIndex(traceManager.getCurrentObject())),
+				RUN_TIMEOUT_MS, RETRY_MS);
+
+			conn.execute("frame select 0");
+			waitForPass(() -> assertEquals("0", frameIndex(traceManager.getCurrentObject())),
+				RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	@Ignore
+	public void testOnSyscallMemory() throws Exception {
+		// TODO: Need a specimen
+		// FWIW, I've already seen this getting exercised in other tests.
+	}
+
+	@Test
+	public void testOnMemoryChanged() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+
+			long address = Long.decode(conn.executeCapture("dis -c1 -n main").split("\\s+")[1]);
+			conn.execute("expr *((char*)(void(*)())main) = 0x7f");
+			conn.execute("ghidra_trace_txstart 'Tx'");
+			conn.execute("ghidra_trace_putmem `(void(*)())main` 10");
+			conn.execute("ghidra_trace_txcommit");
+			
+			waitForPass(() -> {
+				ByteBuffer buf = ByteBuffer.allocate(10);
+				tb.trace.getMemoryManager().getBytes(lastSnap(conn), tb.addr(address), buf);
+				assertEquals(0x7f, buf.get(0));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnRegisterChanged() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+
+			conn.execute("expr $rax = 0x1234");
+			conn.execute("ghidra_trace_txstart 'Tx'");
+			conn.execute("ghidra_trace_putreg");
+			conn.execute("ghidra_trace_txcommit");
+			
+			String path = "Processes[].Threads[].Stack[].Registers";
+			TraceObject registers = Objects.requireNonNull(tb.objAny(path, Lifespan.at(0)));
+			AddressSpace space = tb.trace.getBaseAddressFactory()
+					.getAddressSpace(registers.getCanonicalPath().toString());
+			TraceMemorySpace regs = tb.trace.getMemoryManager().getMemorySpace(space, false);
+			waitForPass(() -> assertEquals("1234",
+				regs.getValue(lastSnap(conn), tb.reg("RAX")).getUnsignedValue().toString(16)));
+		}
+	}
+
+	@Test
+	public void testOnCont() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+
+			conn.execute("cont");
+			waitRunning();
+
+			TraceObject proc = waitForValue(() -> tb.objAny("Processes[]"));
+			waitForPass(() -> {
+				assertEquals("RUNNING", tb.objValue(proc, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnStop() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+
+			TraceObject inf = waitForValue(() -> tb.objAny("Processes[]"));
+			waitForPass(() -> {
+				assertEquals("STOPPED", tb.objValue(inf, lastSnap(conn), "_state"));
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnExited() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			conn.execute("file bash");
+			conn.execute("ghidra_trace_sync_enable");
+			conn.execute("process launch --stop-at-entry -- -c 'exit 1'");
+			txPut(conn, "processes");
+
+			conn.execute("cont");
+			waitRunning();
+			waitStopped();
+
+			waitForPass(() -> {
+				TraceSnapshot snapshot =
+					tb.trace.getTimeManager().getSnapshot(lastSnap(conn), false);
+				assertNotNull(snapshot);
+				assertEquals("Exited with code 1", snapshot.getDescription());
+
+				TraceObject proc = tb.objAny("Processes[]");
+				assertNotNull(proc);
+				Object val = tb.objValue(proc, lastSnap(conn), "_exit_code");
+				assertThat(val, instanceOf(Number.class));
+				assertEquals(1, ((Number) val).longValue());
+			}, RUN_TIMEOUT_MS, RETRY_MS);
+		}
+	}
+
+	@Test
+	public void testOnBreakpointCreated() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+			assertEquals(0, tb.objValues(lastSnap(conn), "Processes[].Breakpoints[]").size());
+
+			conn.execute("breakpoint set -n main");
+			conn.execute("stepi");
+
+			waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Processes[].Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+		}
+	}
+
+	@Test
+	public void testOnBreakpointModified() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+			assertEquals(0, tb.objValues(lastSnap(conn), "Breakpoints[]").size());
+
+			conn.execute("breakpoint set -n main");
+			conn.execute("stepi");
+			TraceObject brk = waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+			assertEquals(null, tb.objValue(brk, lastSnap(conn), "Condition"));
+			conn.execute("breakpoint modify -c 'x>3'");
+			conn.execute("stepi");
+			// NB: Testing "Commands" requires multi-line input - not clear how to do this
+			//assertEquals(null, tb.objValue(brk, lastSnap(conn), "Commands"));
+			//conn.execute("breakpoint command add 'echo test'");
+			//conn.execute("DONE");
+			
+			waitForPass(
+				() -> assertEquals("x>3", tb.objValue(brk, lastSnap(conn), "Condition")));
+		}
+	}
+
+	@Test
+	public void testOnBreakpointDeleted() throws Exception {
+		try (LldbAndTrace conn = startAndSyncLldb()) {
+			start(conn, "bash");
+			assertEquals(0, tb.objValues(lastSnap(conn), "Processes[].Breakpoints[]").size());
+
+			conn.execute("breakpoint set -n main");
+			conn.execute("stepi");
+
+			TraceObject brk = waitForPass(() -> {
+				List<Object> brks = tb.objValues(lastSnap(conn), "Processes[].Breakpoints[]");
+				assertEquals(1, brks.size());
+				return (TraceObject) brks.get(0);
+			});
+
+			conn.execute("breakpoint delete %s".formatted(brk.getCanonicalPath().index()));
+			conn.execute("stepi");
+
+			waitForPass(
+				() -> assertEquals(0, tb.objValues(lastSnap(conn), "Processes[].Breakpoints[]").size()));
+		}
+	}
+	
+	private void start(LldbAndTrace conn, String obj) {
+		conn.execute("file "+obj);
+		conn.execute("ghidra_trace_sync_enable");
+		conn.execute("process launch --stop-at-entry");
+		txPut(conn, "processes");
+	}
+	
+	private void txPut(LldbAndTrace conn, String obj) {
+		conn.execute("ghidra_trace_txstart 'Tx"+obj+"'");
+		conn.execute("ghidra_trace_put_"+obj);
+		conn.execute("ghidra_trace_txcommit");
+	}
+
+}