diff --git a/Ghidra/Features/BSim/certification.manifest b/Ghidra/Features/BSim/certification.manifest index 951b1874fb..480af25ac1 100755 --- a/Ghidra/Features/BSim/certification.manifest +++ b/Ghidra/Features/BSim/certification.manifest @@ -41,6 +41,7 @@ src/main/help/help/topics/BSimSearchPlugin/images/BSimOverviewResults.png||GHIDR src/main/help/help/topics/BSimSearchPlugin/images/BSimResultsProvider.png||GHIDRA||||END| src/main/help/help/topics/BSimSearchPlugin/images/BSimSearchDialog.png||GHIDRA||||END| src/main/help/help/topics/BSimSearchPlugin/images/ManageServersDialog.png||GHIDRA||||END| +src/main/help/help/topics/StructureRecovery/StructureRecovery.html||GHIDRA||||END| src/main/resources/bsim.log4j.xml||GHIDRA||||END| src/main/resources/images/checkmark_yellow.gif||GHIDRA||||END| src/main/resources/images/connected.png||GHIDRA||||END| diff --git a/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml b/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml index ba54898ec6..9f8be0f64d 100755 --- a/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml +++ b/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml @@ -170,6 +170,28 @@ text="BSim Command (bsim)" target="help/topics/BSim/CommandLineReference.html#BSimCommand"/> + + + + + + + + diff --git a/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html b/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html new file mode 100644 index 0000000000..d7b65437cf --- /dev/null +++ b/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html @@ -0,0 +1,203 @@ + + + +Structure Recovery Using BSim + + + + +
+

+Structure Recovery Using BSim

+ +

+ The StructureRecovery plugin allows users to build a structure in a target program based on the + field uses in a source program. The process involves three steps. First, a source program is + mined for field references from a known version of the structure to recover the use addresses + and offsets. Second, functions in the source program bookmarked by field use are matched to + functions in the target program using the local variation of BSim, which does not require a database. + Third, BSim is used to create 'pinnings' between the data-flow graphs for each pair of functions + which identify vertex-vertex correspondences using the decoompiler's diffing algorithm. + Vertices with addresses from the first step are examined to identify offset information, which + then becomes input to the construction of a new structure in the target program. +

+

+ The success of this process depends on a number of factors. There are cases, for instance, in + which nested structures may be missed by the structure use task, and, of course, certain fields + may not be referenced at all. BSim, here as elsewhere, depends on a number of parameters that + determine what constitutes a match, which are exposed as options. Unmatched functions will + potentially result in gaps in the structure. Finally, the pinning comparison attempts to reconcile + disparate pcode representations, but may fail in more complicated scenarios. When successful, + the resulting structure will be stored in the target program's DataTypeManager as "RECOVERED_structure_name". +

+

+ Each of the three tasks is relatively expensive. It may be necessary to save results from each + across sessions. This is done using bookmarks (viewable from the listing or the Bookmarks viewer), which + capture the structure use and function-to-function matches. The former are represented as + "USES_structure_name_offset:address"; the latter as "source function id→target function id". + The function-match bookmarks will be present in both the source and target programs. +

+

+ ALL tasks should run with the source program active. +

+ +
+

+ Principal Actions (Tools → Structure Recovery)

+ +
+

+ Tag Structure Use

+

+ Locates all non-zero field uses within a program for a specified structure. +

+
+ +
+

+ Match Functions

+

+ Uses BSim to match functions with structure use in the source program to similar functions + in a target program. +

+
+ +
+

+ Generate structure

+

+ Compares data-flow 'pinnings' for matched functions and uses the resulting offsets to + reconstruct an approximate match to the original structure for the target progam. +

+
+
+
+ +
+

+ Secondary Actions (Tools → Structure Recovery)

+
+

+ Structure Uses: map to bookmarks

+

+ Converts (persistent) bookmarks containing structure-use information, typically generated + by the 'Tag Structure Use' action, to a (non-persistent) map and vice-versa. +

+
+ +
+

+ Function Matches: map to bookmarks

+ +

+ Converts (persistent) bookmarks containing function-function matches, typically generated + by the 'Match Functions' action, to a (non-persistent) map and vice-versa. +

+ +
+

+ Tool Options (Options/Structure Recovery)

+ +

+ Options governing various aspects of the structure recovery. +

+ +

+ Structure to recover

+

+ The full path to the data type to be replicated ('Tag Structure Use' will query if empty). +

+
+ +

+ Add structure-use bookmarks

+

+ Should bookmarks by generated by 'Tag Structure Use'. +

+
+ +
+

+ Add function-match bookmarks

+

+ Should bookmarks by generated by 'Match Functions'. +

+
+ +
+

+ Match-confidence lower bound

+

+ Used by 'Match Functions' (see BSim documentation for details). +

+
+ +
+

+Match-similarity lower bound

+

+ Used by 'Match Functions' (see BSim documentation for details). +

+
+ +
+

+Match-similarity upper bound

+

+ Used by 'Match Functions' (see BSim documentation for details). +

+
+ +
+

+Self-significance bound

+

+ Used by 'Match Functions' (see BSim documentation for details). +

+
+ +
+

+Min confidence

+

+ Used by 'Match Functions' (see BSim documentation for details). +

+
+ +
+

+Max structure offset

+

+ The largest offset considered valid by 'Generate structure' (prevents confusion + between offsets and addresses). +

+
+ +
+

+Excluded offsets (hex)

+

+ Comma-separated hex values to be ignored by 'Generate structure'. Principally used when + analyzing a randomized structure that amalgamates randomized fields in a common block, + e.g. 0x40 for the Linux task_struct randomized with symbols. +

+
+ +
+

+Match on...

+

+ Should the function-function matching rely on solo matches, solo matches plus matches + exceeding the 'Min confidence', or should a full maximum-weight bipartite match be + computed. +

+
+ +
+ + + + + + + diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java new file mode 100644 index 0000000000..5fcdd053da --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java @@ -0,0 +1,400 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.gui.structs; + +import java.util.*; +import java.util.Map.Entry; + +import db.Transaction; +import ghidra.app.decompiler.*; +import ghidra.app.services.ConsoleService; +import ghidra.features.codecompare.graphanalysis.DataVertex; +import ghidra.features.codecompare.graphanalysis.Pinning; +import ghidra.program.model.address.AddressSet; +import ghidra.program.model.data.*; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.Program; +import ghidra.program.model.pcode.*; +import ghidra.util.Msg; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.Task; +import ghidra.util.task.TaskMonitor; + +public class ComparePinningsTask extends Task { + + private StructureRecoveryPlugin plugin; + private ConsoleService console; + + private static int DECOMPILER_TIMEOUT = 60; + + private final Program currentProgram; + private final Program targetProgram; + private DecompInterface currentInterface; + private DecompInterface targetInterface; + + private Map> decompilerResults = + new HashMap<>(); + private Map> results = new HashMap<>(); + + private Map offsets; + private Map addressesByField; + private Map> functionMap; + private long maxOffset = 0x100000; + + private record PinningMatch(float weight, int sourceOffset, int targetOffset) { + @Override + public String toString() { + return Integer.toHexString(sourceOffset) + ":" + Integer.toHexString(targetOffset) + + " (" + weight + ")"; + } + } + + public ComparePinningsTask(StructureRecoveryPlugin plugin) { + super("Compare Pinnings", true, false, false); + this.plugin = plugin; + this.console = plugin.getConsole(); + this.currentProgram = plugin.getCurrentProgram(); + this.targetProgram = plugin.getTargetProgram(); + this.offsets = plugin.getOffsets(); + this.addressesByField = plugin.getAddressesByField(); + this.functionMap = plugin.getFunctionMap(); + this.maxOffset = plugin.getMaxOffset(); + } + + @Override + public void run(TaskMonitor monitor) { + String taskName = getTaskTitle(); + try { + Thread.currentThread().setName(taskName); + console.addMessage(taskName, "Running..."); + initDecompiler(); + comparePinnings(monitor); + console.addMessage(taskName, "Finished!"); + } + catch (Exception e) { + if (!monitor.isCancelled()) { + Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName + + "\n" + e.getClass().getName() + ": " + e.getMessage(), e); + console.addErrorMessage("", "Error running task: " + taskName); + console.addException(taskName, e); + } + } + finally { + cleanupDecompiler(); + } + } + + protected void comparePinnings(TaskMonitor monitor) throws CancelledException { + monitor.setMessage("Comparing pinnings..."); + monitor.setIndeterminate(false); + monitor.initialize(functionMap.size()); + + for (Function left : functionMap.keySet()) { + monitor.checkCancelled(); + + Set tgtSet = functionMap.get(left); + for (Function right : tgtSet) { + monitor.checkCancelled(); + + DecompileResults currentResults = + getDecompilerResults(currentInterface, left, monitor); + HighFunction currentHFunc = currentResults.getHighFunction(); + if (currentHFunc == null) { + continue; + } + + DecompileResults targetResults = + getDecompilerResults(targetInterface, right, monitor); + HighFunction targetHFunc = targetResults.getHighFunction(); + if (targetHFunc == null) { + continue; + } + + Pinning pin = + Pinning.makePinning(currentHFunc, targetHFunc, false, false, true, + TaskMonitor.DUMMY); + + for (String fieldName : offsets.keySet()) { + Long offset = offsets.get(fieldName); + AddressSet addresses = addressesByField.get(fieldName); + results.putIfAbsent(offset, new HashMap<>()); + analyzePinning(pin, offset, addresses); + } + } + monitor.increment(); + } + //printResults(); + buildStructure(); + + } + + private void initDecompiler() { + DecompileOptions options = new DecompileOptions(); + + currentInterface = new DecompInterface(); + currentInterface.setOptions(options); + currentInterface.openProgram(currentProgram); + + targetInterface = new DecompInterface(); + targetInterface.setOptions(options); + targetInterface.openProgram(targetProgram); + } + + private void cleanupDecompiler() { + if (currentInterface != null) { + currentInterface.dispose(); + } + if (targetInterface != null) { + targetInterface.dispose(); + } + decompilerResults.clear(); + } + + public DecompileResults getDecompilerResults(DecompInterface ifc, Function func, + TaskMonitor monitor) { + return decompilerResults + .computeIfAbsent(ifc.getProgram(), _ -> new HashMap<>()) + .computeIfAbsent(func, f -> ifc.decompileFunction(f, DECOMPILER_TIMEOUT, monitor)); + } + + private void analyzePinning(Pinning pin, long offset, AddressSet addresses) { + Map originalPinMap = pin.getPinMap(); + Map op2vertex = new HashMap<>(); + Map opToVertexMap = new HashMap<>(); + + for (DataVertex lv : originalPinMap.keySet()) { + PcodeOpAST op = lv.getOp(); + if (op != null && !opToVertexMap.containsKey(op)) { + opToVertexMap.put(op, originalPinMap.get(lv)); + op2vertex.put(op, lv); + } + } + + for (PcodeOp lop : opToVertexMap.keySet()) { + DataVertex rv = opToVertexMap.get(lop); + DataVertex lv = op2vertex.get(lop); + if (addresses != null && addresses.contains(lop.getSeqnum().getTarget())) { + compareConstants(lv, rv, offset); + } + } + } + + private void compareConstants(DataVertex vertL, DataVertex vertR, long offset) { + Set constantsL = new LinkedHashSet<>(); + collectConstantsByOperand(vertL.getOp(), new HashSet<>(), constantsL, + vertL.getDepthPaired()); + + Set constantsR = new LinkedHashSet<>(); + collectConstantsByOperand(vertR.getOp(), new HashSet<>(), constantsR, + vertR.getDepthPaired()); + + Set uniqL = extractConstants(constantsL); + Set uniqR = extractConstants(constantsR); + Set init = new TreeSet<>(); + init.addAll(uniqL); + + uniqR.removeAll(plugin.getExcludedOffsets()); + + Set common = new HashSet<>(uniqL); + common.retainAll(uniqR); + + // Remove what's in common + uniqL.removeAll(common); + uniqR.removeAll(common); + + if (uniqL.isEmpty() || uniqR.isEmpty()) { + if (init.contains(offset)) { + upcount(offset, offset); + } + return; + } + + // Use obvious matches + if (uniqL.size() == 1 && uniqL.contains(offset) && uniqR.size() == 1) { + upcount(offset, uniqR.iterator().next()); + } + } + + private void upcount(Long offsetL, Long offsetR) { + results.computeIfAbsent(offsetL, _ -> new HashMap<>()) + .merge(offsetR, 1.0f, Float::sum); + } + + private Set extractConstants(Set constants) { + Set uniq = new TreeSet<>(); + for (Varnode vn : constants) { + PcodeOp descendent = vn.getLoneDescend(); + if (descendent != null) { + int opcode = descendent.getOpcode(); + if (opcode == PcodeOp.INDIRECT) { + continue; + } + if (opcode == PcodeOp.PTRADD) { + Varnode input1 = descendent.getInput(1); + Varnode input2 = descendent.getInput(2); + if (input1.isConstant() && input2.isConstant()) { + uniq.add(Math.abs(input1.getOffset() * input2.getOffset())); + continue; + } + } + } + long vnOffset = vn.getOffset(); + if (vnOffset == 0 || vnOffset == 1) { + continue; + } + vnOffset = Math.abs(vnOffset); + if (vnOffset > maxOffset) { + continue; + } + uniq.add(vnOffset); + } + return uniq; + } + + private void collectConstantsByOperand(PcodeOp op, Set visitedOps, + Set constants, int depth) { + if (depth < 0 || !visitedOps.add(op)) { + return; + } + + int nextDepth = depth - 1; + int numInputs = op.getNumInputs(); + for (int i = 0; i < numInputs; i++) { + Varnode input = op.getInput(i); + if (input != null) { + collectConstantsByVarnode(input, visitedOps, constants, nextDepth); + } + } + } + + private void collectConstantsByVarnode(Varnode vn, Set visitedOps, + Set constants, int depth) { + if (depth < 0) { + return; + } + if (vn.isConstant()) { + constants.add(vn); + } + PcodeOp def = vn.getDef(); + if (def != null) { + collectConstantsByOperand(def, visitedOps, constants, depth); + } + } + + private void buildStructure() { + Structure original = plugin.getTargetDataType(); + if (original == null) { + console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT, + "Error: Target data type is missing."); + return; + } + + List matchHistory = new ArrayList<>(); + long max = -1L; + for (Entry> entry : results.entrySet()) { + Long currentOffset = entry.getKey(); + DataTypeComponent c = original.getComponentAt(currentOffset.intValue()); + int srcOffset = entry.getKey().intValue(); + Map targetMap = entry.getValue(); + for (Entry targetEntry : targetMap.entrySet()) { + Long targetOffset = targetEntry.getKey(); + if (targetOffset > max) { + max = targetOffset + c.getLength(); + } + Float weight = targetEntry.getValue(); + if (weight != null) { + matchHistory.add(new PinningMatch(weight, srcOffset, targetOffset.intValue())); + } + } + } + + matchHistory.sort((a, b) -> Float.compare(b.weight, a.weight)); + + try (Transaction _ = targetProgram.openTransaction("Build structure")) { + ProgramBasedDataTypeManager dtm = targetProgram.getDataTypeManager(); + Structure structure = + new StructureDataType("RECOVERED_" + original.getName(), (int) max, dtm); + + Set placed = new HashSet<>(); + Set occupiedBytes = new HashSet<>(); + for (PinningMatch match : matchHistory) { + if (placed.contains(match.sourceOffset)) { + continue; + } + + DataTypeComponent origComp = original.getComponentAt(match.sourceOffset); + DataTypeComponent targetComp = structure.getComponentAt(match.targetOffset); + + DataType dataType = origComp.getDataType(); + DataType targetDataType = dtm.getDataType(dataType.getDataTypePath()); + + if (targetComp != null && targetComp.isUndefined()) { + if (targetDataType == null) { + // Try to resolve or replicate the type context locally if path missing + targetDataType = + dtm.resolve(dataType, DataTypeConflictHandler.DEFAULT_HANDLER); + } + if (targetDataType == null) { + targetDataType = DataType.DEFAULT; + } + + int componentLength = targetDataType.getLength(); + if (componentLength <= 0) { + componentLength = origComp.getLength(); + } + + // Check for overlapping byte targets (possibly overkill,, but...) + boolean conflict = false; + for (int i = 0; i < componentLength; i++) { + if (occupiedBytes.contains(match.targetOffset + i)) { + conflict = true; + break; + } + } + + if (conflict) { + console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT, + "Overlapping target bounds for " + match + " : " + + origComp + " using Byte"); + targetDataType = ByteDataType.dataType; + componentLength = targetDataType.getLength(); + } + + try { + structure.replaceAtOffset(match.targetOffset, targetDataType, + targetDataType.getLength(), origComp.getFieldName(), + origComp.getComment()); + placed.add(match.sourceOffset); + for (int i = 0; i < componentLength; i++) { + occupiedBytes.add(match.targetOffset + i); + } + } + catch (IllegalArgumentException iae) { + console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT, + iae.getMessage()); + } + } + } + + console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT, + "Placed: " + placed.size() + " in " + structure.getName()); + DataTypeComponent[] definedComponents = structure.getDefinedComponents(); + DataTypeComponent last = definedComponents[definedComponents.length - 1]; + structure.setLength(last.getEndOffset() + 1); + dtm.addDataType(structure, DataTypeConflictHandler.DEFAULT_HANDLER); + } + } +} diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java new file mode 100644 index 0000000000..eb71dab092 --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java @@ -0,0 +1,320 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.gui.structs; + +import java.util.*; + +import org.jgrapht.Graph; +import org.jgrapht.alg.interfaces.MatchingAlgorithm.Matching; +import org.jgrapht.alg.matching.MaximumWeightBipartiteMatching; +import org.jgrapht.graph.DefaultWeightedEdge; +import org.jgrapht.graph.SimpleWeightedGraph; + +import db.Transaction; +import generic.lsh.vector.*; +import ghidra.app.decompiler.DecompileException; +import ghidra.app.services.ConsoleService; +import ghidra.features.bsim.gui.structs.StructureRecoveryPlugin.FunctionMatchOption; +import ghidra.features.bsim.query.*; +import ghidra.features.bsim.query.client.Configuration; +import ghidra.features.bsim.query.description.FunctionDescription; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.*; +import ghidra.util.Msg; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.Task; +import ghidra.util.task.TaskMonitor; + +public class MatchFunctionsTask extends Task { + + //functions with self significance below this bound will be skipped + private double SELF_SIGNIFICANCE_BOUND = 15.0; + //bsim database template determining the signature settings + private String TEMPLATE_NAME = "medium_nosize"; + //these are analogous to the bounds in a bsim query + private double MATCH_SIMILARITY_LOWER_BOUND = 0.2; + //decrease this if you only want to see matches that aren't exact + //for instance, when looking for changes between two versions of a program + private double MATCH_SIMILARITY_UPPER_BOUND = 1.0; + private double MATCH_CONFIDENCE_LOWER_BOUND = 0.0; + + // We'll consider solo matches or matches above this confidence + private double MIN_CONFIDENCE = 20.0; + + private StructureRecoveryPlugin plugin; + private ConsoleService console; + + private Program currentProgram; + private Program targetProgram; + private BookmarkManager srcBookmarks; + private BookmarkManager tgtBookmarks; + + private Set functionsToMatch = new HashSet<>(); + Map> functionMap = new HashMap<>(); + + public final static String TAG = "Function Match"; + + public MatchFunctionsTask(StructureRecoveryPlugin plugin, Program target) { + super("Match Functions", true, false, false); + this.plugin = plugin; + this.targetProgram = target; + this.functionsToMatch = plugin.getFunctionsToMatch(); + this.console = plugin.getConsole(); + this.currentProgram = plugin.getCurrentProgram(); + this.SELF_SIGNIFICANCE_BOUND = plugin.getSelfSiginificanceBound(); + this.MATCH_SIMILARITY_LOWER_BOUND = plugin.getMatchSimilarityLowerBound(); + this.MATCH_SIMILARITY_UPPER_BOUND = plugin.getMatchSimilarityUpperBound(); + this.MATCH_CONFIDENCE_LOWER_BOUND = plugin.getMatchConfidenceLowerBound(); + this.MIN_CONFIDENCE = plugin.getMinConfidence(); + this.srcBookmarks = currentProgram.getBookmarkManager(); + this.tgtBookmarks = targetProgram.getBookmarkManager(); + } + + @Override + public void run(TaskMonitor monitor) { + String taskName = getTaskTitle(); + try { + Thread.currentThread().setName(taskName); + console.addMessage(taskName, "Running..."); + matchFunctions(monitor); + console.addMessage(taskName, "Finished!"); + } + catch (CancelledException e) { + console.addMessage(taskName, "Cancelled by user."); + } + catch (Exception e) { + if (!monitor.isCancelled()) { + Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName + + "\n" + e.getClass().getName() + ": " + e.getMessage(), e); + console.addErrorMessage("", "Error running task: " + taskName); + console.addException(taskName, e); + } + } + } + + private void matchFunctions(TaskMonitor monitor) + throws CancelledException, LSHException, DecompileException { + Iterator functionIter = targetProgram.getFunctionManager().getFunctions(true); + GenSignatures targetSigs = + generateSignatures(targetProgram, functionIter, getVectorFactory(), monitor); + + List matchResults = + getMatchesTwoPrograms(functionsToMatch, targetSigs, monitor); + + FunctionMatchOption matchType = plugin.getOptionSoloMatchesOnly(); + if (matchType != FunctionMatchOption.MAX_WEIGHT) { + defineMapFromSoloMatches(matchResults, matchType, monitor); + } + else { + defineMap(matchResults, monitor); + } + } + + private void defineMapFromSoloMatches(List matchResults, + FunctionMatchOption matchType, TaskMonitor monitor) { + boolean addBookmarks = plugin.getOptionFnMatchBookmarks(); + try (Transaction _ = currentProgram.openTransaction("add"); + Transaction _ = targetProgram.openTransaction("add")) { + Set processed = new HashSet<>(); + for (LocalBSimMatch match : matchResults) { + Function left = match.sourceFunc(); + Set set = functionMap.computeIfAbsent(left, _ -> new HashSet<>()); + Function right = match.targetFunc(); + if (right != null && !matchType.equals(FunctionMatchOption.MAX_WEIGHT)) { + set.add(right); + Double sig = match.significance(); + if (addBookmarks && sig >= MIN_CONFIDENCE) { + addBookmarks(left, right, processed); + } + } + } + for (Function left : functionMap.keySet()) { + Set set = functionMap.get(left); + if (set.size() == 1) { + Function right = set.iterator().next(); + if (addBookmarks) { + addBookmarks(left, right, processed); + } + } + } + } + + plugin.setFunctionMap(functionMap); + } + + private void defineMap(List matchResults, TaskMonitor monitor) + throws CancelledException { + Set setLeft = new HashSet<>(); + Set setRight = new HashSet<>(); + Graph graph = + new SimpleWeightedGraph<>(DefaultWeightedEdge.class); + + monitor.setMessage("Computing local BSim results..."); + monitor.setMaximum(matchResults.size()); + for (LocalBSimMatch match : matchResults) { + monitor.checkCancelled(); + Function left = match.sourceFunc(); + Function right = match.targetFunc(); + if (right != null) { + Double sig = match.significance(); + String leftID = "L" + Long.toString(left.getID()); + String rightID = "R" + Long.toString(right.getID()); + setLeft.add(leftID); + graph.addVertex(leftID); + setRight.add(rightID); + graph.addVertex(rightID); + addWeightedEdge(graph, leftID, rightID, sig); + } + } + + MaximumWeightBipartiteMatching alg = + new MaximumWeightBipartiteMatching<>(graph, setLeft, setRight); + + boolean addBookmarks = plugin.getOptionFnMatchBookmarks(); + try (Transaction _ = currentProgram.openTransaction("add"); + Transaction _ = targetProgram.openTransaction("add")) { + + FunctionManager sourceManager = currentProgram.getFunctionManager(); + FunctionManager targetManager = targetProgram.getFunctionManager(); + monitor.setMessage("Computing match..."); + Matching matching = alg.getMatching(); + + Set processedTags = new HashSet<>(); + monitor.setMessage("Computing map..."); + monitor.setMaximum(matching.getEdges().size()); + for (DefaultWeightedEdge edge : matching.getEdges()) { + monitor.checkCancelled(); + String edgeSource = graph.getEdgeSource(edge).substring(1); + Function left = sourceManager.getFunction(Long.parseLong(edgeSource)); + String edgeTarget = graph.getEdgeTarget(edge).substring(1); + Function right = targetManager.getFunction(Long.parseLong(edgeTarget)); + Set set = functionMap.computeIfAbsent(left, _ -> new HashSet<>()); + set.add(right); + if (addBookmarks) { + addBookmarks(left, right, processedTags); + } + } + } + + plugin.setFunctionMap(functionMap); + } + + private static void addWeightedEdge(Graph graph, String source, + String target, double weight) { + DefaultWeightedEdge edge = graph.addEdge(source, target); + if (edge != null) { + graph.setEdgeWeight(edge, weight); + } + } + + private void addBookmarks(Function left, Function right, Set processed) { + String key = left.getID() + "->" + right.getID(); + if (processed.add(key)) { + srcBookmarks.setBookmark(left.getEntryPoint(), TAG, + Long.toString(left.getID()), Long.toString(right.getID())); + tgtBookmarks.setBookmark(right.getEntryPoint(), TAG, + Long.toString(left.getID()), Long.toString(right.getID())); + } + } + + public record LocalBSimMatch(Function sourceFunc, Function targetFunc, double sim, + double significance) {} + + private List getMatchesTwoPrograms(Set srcFuncs, + GenSignatures targetSigs, TaskMonitor monitor) + throws LSHException, DecompileException, CancelledException { + LSHVectorFactory vectorFactory = getVectorFactory(); + GenSignatures srcSigs = + generateSignatures(currentProgram, srcFuncs.iterator(), vectorFactory, monitor); + Iterator sourceDescripts = + srcSigs.getDescriptionManager().listAllFunctions(); + + VectorCompare vecCompare = new VectorCompare(); + List bsimMatches = new ArrayList<>(); + + List targetList = new ArrayList<>(); + Iterator targetDescriptsIter = + targetSigs.getDescriptionManager().listAllFunctions(); + while (targetDescriptsIter.hasNext()) { + monitor.checkCancelled(); + targetList.add(targetDescriptsIter.next()); + } + + while (sourceDescripts.hasNext()) { + monitor.checkCancelled(); + FunctionDescription srcDesc = sourceDescripts.next(); + + //skip if self-significance too small + LSHVector srcVector = srcDesc.getSignatureRecord().getLSHVector(); + if (vectorFactory.getSelfSignificance(srcVector) <= SELF_SIGNIFICANCE_BOUND) { + continue; + } + + Function srcFunc = getFunction(currentProgram, srcDesc.getAddress()); + for (FunctionDescription targetDesc : targetList) { + //skip if self-significance too small + LSHVector targetVector = targetDesc.getSignatureRecord().getLSHVector(); + if (vectorFactory.getSelfSignificance(targetVector) <= SELF_SIGNIFICANCE_BOUND) { + continue; + } + double sim = srcVector.compare(targetVector, vecCompare); + double sig = vectorFactory.calculateSignificance(vecCompare); + if (sig >= MATCH_CONFIDENCE_LOWER_BOUND && MATCH_SIMILARITY_LOWER_BOUND <= sim && + sim <= MATCH_SIMILARITY_UPPER_BOUND) { + Function targetFunc = getFunction(targetProgram, targetDesc.getAddress()); + if (targetFunc != null) { + bsimMatches.add(new LocalBSimMatch(srcFunc, targetFunc, sim, sig)); + } + else { + Msg.warn(this, "Function not found for " + targetDesc.getAddress()); + } + } + monitor.checkCancelled(); + } + } + return bsimMatches; + } + + private Function getFunction(Program program, long offset) { + Address addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(offset); + return program.getFunctionManager().getFunctionAt(addr); + } + + private LSHVectorFactory getVectorFactory() throws LSHException { + LSHVectorFactory vectorFactory = FunctionDatabase.generateLSHVectorFactory(); + Configuration config = FunctionDatabase.loadConfigurationTemplate(TEMPLATE_NAME); + vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings); + return vectorFactory; + } + + private GenSignatures generateSignatures(Program program, Iterator funcs, + LSHVectorFactory vectorFactory, TaskMonitor monitor) + throws LSHException, DecompileException { + GenSignatures gensig = null; + try { + int count = program.getFunctionManager().getFunctionCount(); + gensig = new GenSignatures(false); + gensig.setVectorFactory(vectorFactory); + gensig.openProgram(program, null, null, null, null, null); + gensig.scanFunctions(funcs, count, monitor); + return gensig; + } + finally { + gensig.dispose(); + } + } + +} diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java new file mode 100644 index 0000000000..d532d44c8c --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java @@ -0,0 +1,115 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.gui.structs; + +import java.util.*; + +import ghidra.app.plugin.core.navigation.locationreferences.LocationReference; +import ghidra.app.plugin.core.navigation.locationreferences.ReferenceUtils; +import ghidra.app.services.ConsoleService; +import ghidra.app.services.FieldMatcher; +import ghidra.program.model.address.Address; +import ghidra.program.model.address.AddressSet; +import ghidra.program.model.data.Structure; +import ghidra.program.model.listing.*; +import ghidra.util.Msg; +import ghidra.util.datastruct.SetAccumulator; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.Task; +import ghidra.util.task.TaskMonitor; + +public class RetrieveUsesTask extends Task { + + private StructureRecoveryPlugin plugin; + private Structure targetDataType; + private ConsoleService console; + private Program currentProgram; + + public final static String TAG = "Structure Use"; + + public RetrieveUsesTask(StructureRecoveryPlugin plugin) { + super("Retrieve Structure Uses", true, false, false); + this.plugin = plugin; + this.targetDataType = plugin.getTargetDataType(); + this.console = plugin.getConsole(); + this.currentProgram = plugin.getCurrentProgram(); + } + + @Override + public void run(TaskMonitor monitor) { + String taskName = getTaskTitle(); + Thread.currentThread().setName(taskName); + try { + console.addMessage(taskName, "Running..."); + retrieveReferences(monitor); + console.addMessage(taskName, "Finished!"); + } + catch (CancelledException e) { + console.addMessage(taskName, "Cancelled by user."); + } + catch (Exception e) { + if (!monitor.isCancelled()) { + Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName + + "\n" + e.getClass().getName() + ": " + e.getMessage(), e); + console.addErrorMessage("", "Error running task: " + taskName); + console.addException(taskName, e); + } + } + } + + protected void retrieveReferences(TaskMonitor monitor) throws CancelledException { + FieldMatcher fieldMatcher = new FieldMatcher(targetDataType); + + SetAccumulator accumulator = new SetAccumulator<>(); + ReferenceUtils.findDataTypeFieldReferences(accumulator, fieldMatcher, currentProgram, true, + monitor); + + Set toMatch = new HashSet<>(); + Map> byField = new HashMap<>(); + Map toAddressSet = new HashMap<>(); + + boolean useBookmarks = plugin.getOptionStructUseBookmarks(); + currentProgram.withTransaction("Add Structure Use Bookmarks", () -> { + BookmarkManager bookmarkManager = currentProgram.getBookmarkManager(); + FunctionManager functionManager = currentProgram.getFunctionManager(); + String typeName = targetDataType.getName(); + + for (LocationReference lr : accumulator.get()) { + String fieldName = lr.getFieldName(); + if (fieldName == null) { + continue; + } + Address locationOfUse = lr.getLocationOfUse(); + Function f = functionManager.getFunctionContaining(locationOfUse); + if (f == null) { + continue; + } + toAddressSet.computeIfAbsent(fieldName, _ -> new AddressSet()) + .add(locationOfUse); + byField.computeIfAbsent(fieldName, _ -> new HashSet()).add(f); + toMatch.add(f); + if (useBookmarks) { + bookmarkManager.setBookmark(locationOfUse, TAG, + "USES_" + typeName, fieldName); + } + } + }); + + plugin.setFunctionsToMatch(toMatch); + plugin.setFunctionsByField(byField); + plugin.setAddressesByField(toAddressSet); + } +} diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java new file mode 100644 index 0000000000..f2c452fd42 --- /dev/null +++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java @@ -0,0 +1,686 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.bsim.gui.structs; + +import static ghidra.framework.main.DataTreeDialogType.*; + +import java.io.IOException; +import java.util.*; +import java.util.Map.Entry; + +import db.Transaction; +import docking.DockingWindowManager; +import docking.action.DockingAction; +import docking.action.MenuData; +import docking.action.builder.ActionBuilder; +import ghidra.app.CorePluginPackage; +import ghidra.app.events.*; +import ghidra.app.merge.structures.StructureMergeDialog; +import ghidra.app.plugin.PluginCategoryNames; +import ghidra.app.plugin.ProgramPlugin; +import ghidra.app.services.ConsoleService; +import ghidra.app.util.datatype.DataTypeSelectionDialog; +import ghidra.framework.main.DataTreeDialog; +import ghidra.framework.model.DomainFile; +import ghidra.framework.model.DomainFileFilter; +import ghidra.framework.options.*; +import ghidra.framework.plugintool.*; +import ghidra.framework.plugintool.util.PluginStatus; +import ghidra.program.model.address.*; +import ghidra.program.model.data.*; +import ghidra.program.model.listing.*; +import ghidra.program.util.ProgramLocation; +import ghidra.util.*; +import ghidra.util.bean.opteditor.OptionsVetoException; +import ghidra.util.data.DataTypeParser.AllowedDataTypes; +import ghidra.util.exception.CancelledException; +import ghidra.util.exception.VersionException; +import ghidra.util.task.TaskMonitor; + +/** + * Plugin for recovering structure layouts across programs. + */ +//@formatter:off +@PluginInfo( + status = PluginStatus.UNSTABLE, + packageName = CorePluginPackage.NAME, + category = PluginCategoryNames.ANALYSIS, + shortDescription = "StructureRecovery", + description = "Plugin for recovering structure layouts across programs", + servicesRequired = { + ConsoleService.class + }, + eventsConsumed = { + ProgramActivatedPluginEvent.class, ProgramOpenedPluginEvent.class, + ProgramLocationPluginEvent.class, ProgramSelectionPluginEvent.class, + ProgramClosedPluginEvent.class + }) +//@formatter:on + +public class StructureRecoveryPlugin extends ProgramPlugin implements OptionsChangeListener { + + static final String TAG_STRUCTURE_USE = "Tag structure use"; + static final String MATCH_FUNCTIONS = "Match functions"; + static final String REGENERATE_STRUCT = "Generate structure"; + static final String COMPARE_STRUCTURES = "Compare new structure"; + + private static final String FUNCTIONS_MAP_BKMKS = "Function matches: map to bookmarks"; + private static final String FUNCTIONS_BKMKS_MAP = "Function matches: bookmarks to map"; + private static final String FUNCTIONS_BKMKS_DEL = "Function matches: remove bookmarks"; + private static final String USES_MAP_BKMKS = "Structure uses: map to bookmarks"; + private static final String USES_BKMKS_MAP = "Structure uses: bookmarks to map"; + private static final String USES_BKMKS_DEL = "Structure uses: remove bookmarks"; + + public final static String HELP_LOCATION = "StructureRecovery"; + + private static final String DEFAULT_STRUCTURE = "Structure to recover"; + private static final String ADD_STRUCTURE_USE_BKMKS = "Add structure-use bookmarks"; + private static final String ADD_FUNCTION_MATCH_BKMKS = "Add function-match bookmarks"; + private static final String SELF_SIGNIFICANCE_BOUND = "Self-significance bound"; + private static final String MATCH_SIMILARITY_LOWER_BOUND = "Match-similarity lower bound"; + private static final String MATCH_SIMILARITY_UPPER_BOUND = "Match-similarity upper bound"; + private static final String MATCH_CONFIDENCE_LOWER_BOUND = "Match-confidence lower bound"; + private static final String MIN_CONFIDENCE = "Min confidence"; + private static final String MAX_OFFSET = "Max structure offset"; + private static final String EXCLUDED_OFFSETS = "Excluded offsets (CSV hex values)"; + private static final String MATCH_ON = "Match on"; + + public enum FunctionMatchOption { + SOLO_MATCHES, SOLO_OR_MIN_CONFIDENCE, MAX_WEIGHT + } + + Program targetProgram; + + Map> functionMap = new HashMap<>(); + Map confidence = new HashMap<>(); + + private Function currentFunction; + private ConsoleService consoleService; + + private Structure targetDataType; + private Map offsets = new HashMap<>(); + private Map names = new HashMap<>(); + private Set srcFunctionsToMatch = new HashSet<>(); + private Map> srcFunctionsByField = new HashMap<>(); + private Map srcAddressesByField = new HashMap<>(); + private ToolOptions options; + private DockingAction tagAction; + private DockingAction regenerateAction; + private DockingAction compareAction; + + public StructureRecoveryPlugin(PluginTool tool) { + super(tool); + createActions(); + getOptions(); + } + + public Function getCurrentFunction() { + return currentFunction; + } + + @Override + protected void programActivated(Program program) { + currentProgram = program; + } + + @Override + protected void programClosed(Program program) { + currentProgram = null; + } + + private void createActions() { + + tagAction = new ActionBuilder(TAG_STRUCTURE_USE, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + if (checkDataType()) { + tool.execute(new RetrieveUsesTask(StructureRecoveryPlugin.this)); + } + }) + .menuGroup("A", "a") + .menuPath("Tools", "Structure recovery", TAG_STRUCTURE_USE) + .buildAndInstall(tool); + + new ActionBuilder(MATCH_FUNCTIONS, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsToMatch.isEmpty()) + .onAction(_ -> { + tool.execute( + new MatchFunctionsTask(StructureRecoveryPlugin.this, getTargetProgram())); + }) + .menuGroup("A", "b") + .menuPath("Tools", "Structure recovery", MATCH_FUNCTIONS) + .buildAndInstall(tool); + + regenerateAction = new ActionBuilder(REGENERATE_STRUCT, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty() && + targetProgram != currentProgram) + .onAction(_ -> { + if (checkDataType()) { + tool.execute(new ComparePinningsTask(StructureRecoveryPlugin.this)); + } + }) + .menuGroup("A", "c") + .menuPath("Tools", "Structure recovery", REGENERATE_STRUCT) + .buildAndInstall(tool); + + compareAction = new ActionBuilder(COMPARE_STRUCTURES, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + compareStructures(); + }) + .menuGroup("A", "d") + .menuPath("Tools", "Structure recovery", COMPARE_STRUCTURES) + .buildAndInstall(tool); + + new ActionBuilder(USES_BKMKS_MAP, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + if (checkDataType()) { + getConsole().addMessage(USES_BKMKS_MAP, "Running..."); + regenerateUsesMapFromBookmarks(); + getConsole().addMessage(USES_BKMKS_MAP, "Finished!"); + } + }) + .menuGroup("B", "a") + .menuPath("Tools", "Structure recovery", USES_BKMKS_MAP) + .buildAndInstall(tool); + + new ActionBuilder(USES_MAP_BKMKS, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsByField.isEmpty()) + .onAction(_ -> { + getConsole().addMessage(USES_MAP_BKMKS, "Running..."); + generateUsesMapBookmarks(); + getConsole().addMessage(USES_MAP_BKMKS, "Finished!"); + }) + .menuGroup("B", "b") + .menuPath("Tools", "Structure recovery", USES_MAP_BKMKS) + .buildAndInstall(tool); + + new ActionBuilder(FUNCTIONS_BKMKS_MAP, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Running..."); + regenerateFunctionMapFromBookmarks(); + getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Finished!"); + }) + .menuGroup("B", "c") + .menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_MAP) + .buildAndInstall(tool); + + new ActionBuilder(FUNCTIONS_MAP_BKMKS, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty()) + .onAction(_ -> { + getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Running..."); + generateFunctionMapBookmarks(); + getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Finished!"); + }) + .menuGroup("B", "d") + .menuPath("Tools", "Structure recovery", FUNCTIONS_MAP_BKMKS) + .buildAndInstall(tool); + + new ActionBuilder(USES_BKMKS_DEL, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + wipeUsesMapBookmarks(); + }) + .menuGroup("B", "e") + .menuPath("Tools", "Structure recovery", USES_BKMKS_DEL) + .buildAndInstall(tool); + + new ActionBuilder(FUNCTIONS_BKMKS_DEL, HELP_LOCATION) + .enabledWhen(_ -> getCurrentProgram() != null) + .onAction(_ -> { + wipeFunctionMapBookmarks(); + }) + .menuGroup("B", "f") + .menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_DEL) + .buildAndInstall(tool); + + } + + @Override + public void optionsChanged(ToolOptions options, String optionName, Object oldValue, + Object newValue) throws OptionsVetoException { + if (optionName.equals(DEFAULT_STRUCTURE)) { + checkDataType(); + boolean valid = targetDataType != null; + String tagCmd = !valid ? TAG_STRUCTURE_USE : "Tag " + targetDataType.getName() + " use"; + String regenCmd = !valid ? REGENERATE_STRUCT + : "Generate " + targetDataType.getName(); + if (targetProgram != null) { + regenCmd += " (" + targetProgram + ")"; + } + String compCmd = !valid ? COMPARE_STRUCTURES + : "Compare " + targetDataType.getName() + " structures"; + tagAction.setMenuBarData( + new MenuData(new String[] { "Tools", "Structure recovery", tagCmd })); + regenerateAction.setMenuBarData(new MenuData( + new String[] { "Tools", "Structure recovery", regenCmd })); + compareAction.setMenuBarData(new MenuData( + new String[] { "Tools", "Structure recovery", compCmd })); + } + } + + @Override + public Program getCurrentProgram() { + return currentProgram; + } + + @Override + public void close() { + if (targetProgram != null) { + targetProgram.release(this); + } + } + + @Override + public void processEvent(PluginEvent event) { + super.processEvent(event); + + //Msg.info(this, "TaintPlugin -> processEvent: " + event.toString() ); + + if (event instanceof ProgramClosedPluginEvent) { + if (currentProgram != null) { + currentProgram.release(this); + } + return; + } + + if (event instanceof ProgramActivatedPluginEvent) { + currentProgram = ((ProgramActivatedPluginEvent) event).getActiveProgram(); + } + else if (event instanceof ProgramLocationPluginEvent) { + ProgramLocation location = ((ProgramLocationPluginEvent) event).getLocation(); + Address address = location.getAddress(); + + if (address.isExternalAddress()) { + // ignore external functions when it comes to taint. + return; + } + + if (currentProgram != null) { + // The user loaded a program for analysis. + Listing listing = currentProgram.getListing(); + Function f = listing.getFunctionContaining(address); + // We are in function f + if (currentFunction == null || !currentFunction.equals(f)) { + currentFunction = f; + } + } + } + } + + private boolean checkDataType() { + if (targetDataType != null) { + return true; + } + if (currentProgram == null) { + Msg.showWarn(this, null, "No Program", + "Please open a program before selecting a structure."); + return false; + } + DataTypeManager dtm = currentProgram.getDataTypeManager(); + String structureName = getDefaultStructureName(); + DataType dataType = null; + if (!structureName.equals("")) { + dataType = dtm.getDataType(structureName); + if (dataType == null) { + List list = new ArrayList<>(); + dtm.findDataTypes(structureName, list); + if (list.isEmpty()) { + Msg.error(this, dataType + " in ToolOptions is not valid."); + return false; + } + dataType = list.get(0); + } + } + else { + DataTypeSelectionDialog selectionDialog = + new DataTypeSelectionDialog(tool, dtm, -1, AllowedDataTypes.FIXED_LENGTH); + tool.showDialog(selectionDialog); + dataType = selectionDialog.getUserChosenDataType(); + if (dataType == null) { + return false; + } + options.setString(DEFAULT_STRUCTURE, dataType.getPathName()); + } + if (dataType instanceof Structure struct) { + targetDataType = struct; + offsets.clear(); + names.clear(); + for (DataTypeComponent dtc : targetDataType.getComponents()) { + if (dtc.getFieldName() != null && dtc.getOffset() > 0) { + offsets.put(dtc.getFieldName(), (long) dtc.getOffset()); + names.put((long) dtc.getOffset(), dtc.getFieldName()); + } + } + return true; + } + return false; + } + + public Program loadTargetProgram() throws VersionException, IOException, CancelledException { + DomainFileFilter filter = df -> Program.class.isAssignableFrom(df.getDomainObjectClass()); + DataTreeDialog dtd = new DataTreeDialog(null, "Target Program", OPEN, filter); + dtd.show(); + if (dtd.wasCancelled()) { + return null; + } + + DomainFile df = dtd.getDomainFile(); + return (Program) df.getDomainObject(this, true, false, TaskMonitor.DUMMY); + } + + private void generateFunctionMapBookmarks() { + BookmarkManager srcBookmarks = currentProgram.getBookmarkManager(); + BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager(); + try (Transaction _ = currentProgram.openTransaction("add")) { + try (Transaction _ = targetProgram.openTransaction("add")) { + for (Entry> entry : functionMap.entrySet()) { + Function left = entry.getKey(); + for (Function right : entry.getValue()) { + srcBookmarks.setBookmark(left.getEntryPoint(), MatchFunctionsTask.TAG, + Long.toString(left.getID()), Long.toString(right.getID())); + tgtBookmarks.setBookmark(right.getEntryPoint(), MatchFunctionsTask.TAG, + Long.toString(left.getID()), Long.toString(right.getID())); + } + } + } + } + } + + private void regenerateFunctionMapFromBookmarks() { + getTargetProgram(); + FunctionManager srcMgr = currentProgram.getFunctionManager(); + FunctionManager tgtMgr = targetProgram.getFunctionManager(); + boolean adds = false; + BookmarkManager bkmkMgr = currentProgram.getBookmarkManager(); + Iterator iterator = bkmkMgr.getBookmarksIterator(MatchFunctionsTask.TAG); + while (iterator.hasNext()) { + Bookmark mark = iterator.next(); + try { + Function left = srcMgr.getFunction(Long.parseLong(mark.getCategory())); + if (left == null) { + continue; + } + Set set = functionMap.computeIfAbsent(left, _ -> new HashSet<>()); + String tgtId = mark.getComment(); + Function right = tgtMgr.getFunction(Long.parseLong(tgtId)); + if (right == null) { + continue; + } + set.add(right); + adds = true; + } + catch (NumberFormatException e) { + // Skip + } + } + if (!adds) { + getConsole().addErrorMessage(FUNCTIONS_BKMKS_MAP, + "Map empty - check function id bookmarks"); + } + } + + private void wipeFunctionMapBookmarks() { + try (Transaction _ = currentProgram.openTransaction("wipeCurrent")) { + getTargetProgram(); + BookmarkManager srcBookmarks = currentProgram.getBookmarkManager(); + srcBookmarks.removeBookmarks(MatchFunctionsTask.TAG); + } + getTargetProgram(); + try (Transaction _ = targetProgram.openTransaction("wipeTarget")) { + getTargetProgram(); + BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager(); + tgtBookmarks.removeBookmarks(MatchFunctionsTask.TAG); + } + } + + private void generateUsesMapBookmarks() { + BookmarkManager bookmarkManager = currentProgram.getBookmarkManager(); + try (Transaction _ = currentProgram.openTransaction("add")) { + for (Entry> entry : srcFunctionsByField.entrySet()) { + String fname = entry.getKey(); + AddressSet addressSet = srcAddressesByField.get(fname); + if (addressSet == null) { + continue; + } + Iterator iterator = entry.getValue().iterator(); + while (iterator.hasNext()) { + Function f = iterator.next(); + AddressSet intersect = addressSet.intersect(f.getBody()); + if (intersect == null) { + continue; + } + // Add a bookmark for each intersecting address range's minimum address so + // it can be parsed back into a single address when rebuilding the map. + for (AddressRange range : intersect.getAddressRanges()) { + bookmarkManager.setBookmark(range.getMinAddress(), RetrieveUsesTask.TAG, + "USES_" + targetDataType.getName(), fname); + } + } + } + } + } + + private void regenerateUsesMapFromBookmarks() { + FunctionManager srcMgr = currentProgram.getFunctionManager(); + BookmarkManager bkmkMgr = currentProgram.getBookmarkManager(); + Iterator iterator = bkmkMgr.getBookmarksIterator(RetrieveUsesTask.TAG); + while (iterator.hasNext()) { + Bookmark mark = iterator.next(); + AddressSet aset = + srcAddressesByField.computeIfAbsent(mark.getComment(), _ -> new AddressSet()); + Address address = mark.getAddress(); + if (address != null) { + aset.add(address); + } + Set set = + srcFunctionsByField.computeIfAbsent(mark.getComment(), _ -> new HashSet<>()); + Function f = srcMgr.getFunctionContaining(address); + if (f != null) { + set.add(f); + srcFunctionsToMatch.add(f); + } + } + if (srcFunctionsToMatch.isEmpty()) { + getConsole().addErrorMessage(USES_BKMKS_MAP, "Map empty - check USES bookmarks"); + } + } + + private void wipeUsesMapBookmarks() { + try (Transaction _ = currentProgram.openTransaction("wipe")) { + BookmarkManager bkmkMgr = currentProgram.getBookmarkManager(); + bkmkMgr.removeBookmarks(RetrieveUsesTask.TAG); + } + } + + private void compareStructures() { + checkDataType(); + getTargetProgram(); + if (targetProgram == null) { + consoleService.addErrorMessage("Compare Structures", "Target program is null."); + return; + } + DataTypeManager dtm = targetProgram.getDataTypeManager(); + String structureName = "/RECOVERED_" + targetDataType.getName(); + DataType dataType = dtm.getDataType(structureName); + if (dataType == null) { + consoleService.addErrorMessage("Compare Structures", structureName + " not found."); + return; + } + if (dataType instanceof Structure recoveredStructure) { + StructureMergeDialog dialog = + new StructureMergeDialog("Compare Structures", recoveredStructure, targetDataType, + _ -> { + //IGNORE + }); + DockingWindowManager.showDialog(dialog); + } + } + + public Map getOffsets() { + return offsets; + } + + public Map getFieldNames() { + return names; + } + + public Map> getFunctionMap() { + return functionMap; + } + + public void setFunctionMap(Map> map) { + this.functionMap = map; + } + + public Map> getFunctionsByField() { + return srcFunctionsByField; + } + + public void setFunctionsByField(Map> functionsByField) { + this.srcFunctionsByField = functionsByField; + } + + public Map getAddressesByField() { + return srcAddressesByField; + } + + public void setAddressesByField(Map toAddressSet) { + this.srcAddressesByField = toAddressSet; + } + + public Set getFunctionsToMatch() { + return srcFunctionsToMatch; + } + + public void setFunctionsToMatch(Set srcFunctionsToMatch) { + this.srcFunctionsToMatch = srcFunctionsToMatch; + } + + public Structure getTargetDataType() { + return targetDataType; + } + + public ConsoleService getConsole() { + if (consoleService == null) { + consoleService = tool.getService(ConsoleService.class); + } + return consoleService; + } + + public Program getTargetProgram() { + if (targetProgram == null) { + Swing.runNow(() -> { + try { + targetProgram = loadTargetProgram(); + } + catch (VersionException | CancelledException | IOException e) { + getConsole().addMessage("Load target", e.getMessage()); + } + }); + } + return targetProgram; + } + + private void getOptions() { + options = tool.getOptions("Structure Recovery"); + HelpLocation help = new HelpLocation(getName(), "Options"); + options.registerOption(DEFAULT_STRUCTURE, OptionType.STRING_TYPE, "", help, + DEFAULT_STRUCTURE); + options.registerOption(ADD_STRUCTURE_USE_BKMKS, OptionType.BOOLEAN_TYPE, true, help, + "Generate structure-use function bookmarks by default"); + options.registerOption(ADD_FUNCTION_MATCH_BKMKS, OptionType.BOOLEAN_TYPE, true, help, + "Generate function-match function bookmarks by default"); + options.registerOption(SELF_SIGNIFICANCE_BOUND, OptionType.DOUBLE_TYPE, 15.0, help, + "Functions with self significance below this bound will be skipped"); + options.registerOption(MATCH_SIMILARITY_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.3, help, + "Lower threshold equates to looser matches"); + options.registerOption(MATCH_SIMILARITY_UPPER_BOUND, OptionType.DOUBLE_TYPE, 1.0, help, + "Decrease this if you only want to see matches that aren't exact"); + options.registerOption(MATCH_CONFIDENCE_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.0, help, + MATCH_CONFIDENCE_LOWER_BOUND); + options.registerOption(MIN_CONFIDENCE, OptionType.DOUBLE_TYPE, 20.0, help, + "Consider matches above this confidence"); + options.registerOption(MAX_OFFSET, OptionType.LONG_TYPE, 100000L, help, + "Ignore constants above this value as probable addresses"); + options.registerOption(EXCLUDED_OFFSETS, OptionType.STRING_TYPE, "", help, + "Useful for structures using boxed randomized fields"); + options.registerOption(MATCH_ON, OptionType.ENUM_TYPE, + FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE, help, + "Match using solo matches, solo+confidence, bipartite pairing"); + options.addOptionsChangeListener(this); + } + + public String getDefaultStructureName() { + return options.getString(DEFAULT_STRUCTURE, ""); + } + + public boolean getOptionStructUseBookmarks() { + return options.getBoolean(ADD_STRUCTURE_USE_BKMKS, true); + } + + public boolean getOptionFnMatchBookmarks() { + return options.getBoolean(ADD_FUNCTION_MATCH_BKMKS, true); + } + + public double getSelfSiginificanceBound() { + return options.getDouble(SELF_SIGNIFICANCE_BOUND, 15.0); + } + + public double getMatchSimilarityLowerBound() { + return options.getDouble(MATCH_SIMILARITY_LOWER_BOUND, 0.3); + } + + public double getMatchSimilarityUpperBound() { + return options.getDouble(MATCH_SIMILARITY_UPPER_BOUND, 1.0); + } + + public double getMatchConfidenceLowerBound() { + return options.getDouble(MATCH_CONFIDENCE_LOWER_BOUND, 0.0); + } + + public double getMinConfidence() { + return options.getDouble(MIN_CONFIDENCE, 20.0); + } + + public long getMaxOffset() { + return options.getLong(MAX_OFFSET, 100000); + } + + public Set getExcludedOffsets() { + String offstr = options.getString(EXCLUDED_OFFSETS, ""); + String[] split = offstr.split(","); + Set excluded = new HashSet<>(); + for (String offset : split) { + if (!offset.equals("")) { + try { + excluded.add(Long.parseLong(offset.trim(), 16)); + } + catch (NumberFormatException e) { + // Skip + } + } + } + return excluded; + } + + public FunctionMatchOption getOptionSoloMatchesOnly() { + return options.getEnum(MATCH_ON, FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE); + } + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java b/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java index 1b02584827..c2d280754f 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -84,4 +84,5 @@ public interface DataTypeReferenceFinder extends ExtensionPoint { */ public void findReferences(Program program, FieldMatcher fieldMatcher, Consumer callback, TaskMonitor monitor) throws CancelledException; + } diff --git a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java index c07918cf81..30960505fb 100755 --- a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java +++ b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -212,4 +212,34 @@ public class DataVertex { } return PcodeOp.isCommutative(opc); } + + public PcodeOpAST getOp() { + return op; + } + + public VarnodeAST getVarnode() { + return vn; + } + + public ArrayList getSources() { + return sources; + } + + /** + * Get the depth at which the pairing occurred + * @return depth of pairing + */ + public int getDepthPaired() { + int size = ngrams.size(); + if (size <= 0) { + return 0; + } + DataNGram gramPaired = ngrams.get(size - 1); + for (DataNGram gram : ngrams) { + if (gram.hash == gramPaired.hash) { + return gram.depth; + } + } + return 0; + } } diff --git a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java index b6e255e13b..0c65fd85b3 100755 --- a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java +++ b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java @@ -908,4 +908,8 @@ public class Pinning { } return result; } + + public Map getPinMap() { + return pinMap; + } }