GP-6309 Refactor SequenceSearchState into BulkPatternSearcher

This commit is contained in:
ghidragon
2026-01-13 13:44:39 -05:00
parent 45252a5b15
commit e1e10c27b7
29 changed files with 1156 additions and 697 deletions

View File

@@ -60,9 +60,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
private final static boolean OPTION_DEFAULT_BOOKMARKS = false;
private static ProgramDecisionTree patternDecisitionTree;
// always need to initialize the root.
SequenceSearchState<Pattern> rootState = null;
SequenceSearchState<Pattern> explicitState = null; //for use during dynamic function start pattern discovery
// always need to initialize the pattern searcher.
BulkPatternSearcher<Pattern> patternSearcher = null;
BulkPatternSearcher<Pattern> explicitSearcher = null; //for use during dynamic function start pattern discovery
private boolean executableBlocksOnly = true; // true if we only analyze executable blocks
@@ -91,7 +91,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
}
return patternDecisitionTree;
}
public ProgramDecisionTree getPatternDecisionTree() {
return initializePatternDecisionTree();
}
@@ -104,39 +104,32 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
this(name, DESCRIPTION, analyzerType);
}
public FunctionStartAnalyzer(String name, String description, AnalyzerType analyzerType) {
super(name, description, analyzerType);
setPriority(AnalysisPriority.CODE_ANALYSIS.after().after());
setDefaultEnablement(true);
setSupportsOneTimeAnalysis();
}
/**
* Sets the {@link SequenceSearchState}. Use this method when you've created a
* {@link SequenceSearchState} that you want to apply to the program. If you don't set
* the state explicitly, Ghidra will create one from the appropriate pattern file in
* {@link SequenceSearchState#initialize}
* @param explicit
* Sets the {@link BulkPatternSearcher}. Use this method when you've created a
* {@link BulkPatternSearcher} that you want to apply to the program. If you don't set
* the state explicitly, Ghidra will create one from the appropriate pattern file.
* @param searcher the explicit BulkPatternSearcher to use
*/
public void setExplicitState(SequenceSearchState<Pattern> explicit) {
explicitState = explicit;
public void setExplicitState(BulkPatternSearcher<Pattern> searcher) {
explicitSearcher = searcher;
}
/**
* Clears the explict state.
* Clears the explicit state.
*/
public void clearExplicitState() {
explicitState = null;
explicitSearcher = null;
}
/**
* apply any latent context at the location
*
* @param program
* @param addr
*/
private void setCurrentContext(Program program, Address addr) {
if (contextValueList == null) {
return;
@@ -159,7 +152,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
contextValueList = null;
}
private void setDisassemblerContext(Program program, PseudoDisassemblerContext pcont, Address addr) {
private void setDisassemblerContext(Program program, PseudoDisassemblerContext pcont,
Address addr) {
if (contextValueList == null) {
return;
}
@@ -203,7 +197,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
private static final int MUST_HAVE_VALID_INSTRUCTIONS_NO_MIN = -1; // no minimum
private static final int VALID_INSTRUCTIONS_NO_MAX = -1; // no maximum on instructions to check
private static final int NO_VALID_INSTRUCTIONS_REQUIRED = 0;
private String afterName = null;
private int validCodeMin = NO_VALID_INSTRUCTIONS_REQUIRED;
private int validCodeMax = VALID_INSTRUCTIONS_NO_MAX;
@@ -233,12 +227,12 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
if (block == null) {
return false;
}
Matcher m = sectionNamePattern.matcher(block.getName());
if (!m.matches()) {
Matcher m = sectionNamePattern.matcher(block.getName());
if (!m.matches()) {
return false;
}
}
/**
* If the match's mark point occurs in undefined data, schedule disassembly
* and a function start at that address. If the match's mark point occurs at an instruction, but that
@@ -264,21 +258,23 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
PseudoDisassembler pseudoDisassembler = new PseudoDisassembler(program);
PseudoDisassemblerContext pcont =
new PseudoDisassemblerContext(program.getProgramContext());
setDisassemblerContext(program, pcont, addr);
boolean isvalid = false;
if (validCodeMin == -1) {
if (validCodeMax > 0) { // check at most N instructions
pseudoDisassembler.setMaxInstructions(validCodeMax);
}
isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, true, contiguous);
isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, true,
contiguous);
}
else {
if (validCodeMax > 0) { // check at most N instructions
pseudoDisassembler.setMaxInstructions(validCodeMax);
}
// disassemble only fallthru, must have validcode number of instructions
isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, false, contiguous);
isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, false,
contiguous);
int instrCount = pseudoDisassembler.getLastCheckValidInstructionCount();
if (instrCount < validCodeMin) {
isvalid = false;
@@ -443,7 +439,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
// if there are only pure data references to the location
return pureDataReferencesOnly(program, addr);
}
}
return true;
}
@@ -456,7 +452,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
* @return true if there are only pure data references (no flow, or r/w)
*/
private boolean pureDataReferencesOnly(Program program, Address addrToCheck) {
ReferenceIterator referencesTo = program.getReferenceManager().getReferencesTo(addrToCheck);
ReferenceIterator referencesTo =
program.getReferenceManager().getReferencesTo(addrToCheck);
if (!referencesTo.hasNext()) {
return false;
}
@@ -486,12 +483,13 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
Function funcAbove = getFunctionAbove(program, addr);
return checkAlreadyInFunctionAbove(program, addr, funcAbove);
}
/*
* Check if in a function above
* return true if already in function above, false otherwise even if in another function
*/
private boolean checkAlreadyInFunctionAbove(Program program, Address addr, Function funcAbove) {
private boolean checkAlreadyInFunctionAbove(Program program, Address addr,
Function funcAbove) {
// if no funcAbove, make sure an instruction, doesn't fall into this one.
Address addrBefore = addr.previous();
if (addrBefore == null) {
@@ -530,13 +528,13 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
return false;
}
/**
* Get an existing function right above the addr.
* @param program program to check
* @param addr address to check
* @return true if there is an existing function above addr
*/
*/
private Function getFunctionAbove(Program program, Address addr) {
// make sure there is an end of function before this one, and addr is not in the function
Function func = null;
@@ -570,7 +568,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
String attrValue = attributes.get(attrName);
attrName = attrName.toLowerCase();
switch (attrName) {
case "after":
case "after":
afterName = attrValue;
if (afterName.startsWith("func")) {
hasCodeConstraints = true;
@@ -592,8 +590,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
"funcstart pattern attribute 'after' must be one of 'function', 'instruction', 'data', 'defined'");
}
break;
// set check for valid code and the minimum number of instructions required
// set check for valid code and the minimum number of instructions required
// if no maximum is set, then the instructions MUST be fallthru instructions, don't check branch flows
case "validcode":
String validcodeStr = attrValue;
@@ -617,9 +615,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
validCodeMax = validCodeMin;
}
break;
// set the maximum number of instructions to check
// if maximum is set, then allow non fallthru instructions while flowing
// set the maximum number of instructions to check
// if maximum is set, then allow non fallthru instructions while flowing
case "validcodemax":
String validcodeMaxStr = attrValue;
// check up <N> instructions for valid code
@@ -630,39 +628,41 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
validCodeMin = MUST_HAVE_VALID_INSTRUCTIONS_NO_MIN;
}
break;
// minimum number of instructions for validcode must be contiguous instructions
case "contiguous":
String fallThruOnlyStr = attrValue;
// check up <N> instructions for valid code
String fallThruOnlyStr = attrValue;
// check up <N> instructions for valid code
contiguous = true;
if (fallThruOnlyStr.equalsIgnoreCase("false")) {
contiguous = false;
}
else if (fallThruOnlyStr.equalsIgnoreCase("true")) {
contiguous = true;
if (fallThruOnlyStr.equalsIgnoreCase("false")) {
contiguous = false;
}
else if (fallThruOnlyStr.equalsIgnoreCase("true")) {
contiguous = true;
} else {
Msg.error(this, "Bad contiguous option (true,false): " + attrName + " = " + attrValue);
}
break;
}
else {
Msg.error(this, "Bad contiguous option (true,false): " + attrName +
" = " + attrValue);
}
break;
case "label":
String name = attrValue;
label = name;
break;
case "thunk":
isThunk = true;
break;
case "section":
sectionNamePattern = java.util.regex.Pattern.compile(attrValue);
break;
case "noreturn":
noreturn = true;
break;
// TODO: add the ability to make data based on a pattern of bytes
// useful after defined instructions/functions to take up filler byte patterns
// will allow more finding of code that is after defined data
@@ -671,7 +671,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
// // create undefined data of the given size
// makeData = Integer.parseInt(validcodeDataStr);
// break;
default:
Msg.error(this, "Unknown Patten option: " + attrName + " = " + attrValue);
}
@@ -760,7 +760,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
}
}
@Override
public boolean canAnalyze(Program program) {
ProgramDecisionTree patternDecisionTree = getPatternDecisionTree();
@@ -795,8 +795,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log)
throws CancelledException {
SequenceSearchState<Pattern> root = initialize(program);
if (root == null) {
BulkPatternSearcher<Pattern> searcher = initialize(program);
if (searcher == null) {
String message = "Could not initialize a search state.";
log.appendMsg(getName(), message);
log.setStatus(message);
@@ -810,27 +810,27 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
// this will keep cruft from accumulating in the property map.
getOrCreatePotentialMatchPropertyMap(program).remove(set);
MemoryBytePatternSearcher patternSearcher;
patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) {
MemoryBytePatternSearcher memorySearcher =
new MemoryBytePatternSearcher("Function Starts", searcher) {
@Override
public void preMatchApply(MatchAction[] actions, Address addr) {
contextValueList = null; // make sure, only context from these actions used
}
@Override
public void postMatchApply(MatchAction[] actions, Address addr) {
// Actions might have set context, check if postcondition failed first
if (!postreqFailedResult.contains(addr)) {
setCurrentContext(program, addr);
@Override
public void preMatchApply(MatchAction[] actions, Address addr) {
contextValueList = null; // make sure, only context from these actions used
}
// get rid of the context list.
contextValueList = null;
}
};
patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly);
patternSearcher.search(program, set, monitor);
@Override
public void postMatchApply(MatchAction[] actions, Address addr) {
// Actions might have set context, check if postcondition failed first
if (!postreqFailedResult.contains(addr)) {
setCurrentContext(program, addr);
}
// get rid of the context list.
contextValueList = null;
}
};
memorySearcher.setSearchExecutableOnly(doExecutableBlocksOnly);
memorySearcher.search(program, set, monitor);
AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program);
if (!disassemResult.isEmpty()) {
@@ -838,7 +838,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
AddressSet doNowDisassembly = disassemResult.intersect(funcResult);
// this will disassemble at this analyzers priority
analysisManager.disassemble(doNowDisassembly);
// delay disassemble of possible function starts
AddressSet delayedDisassembly = disassemResult.subtract(funcResult);
analysisManager.disassemble(delayedDisassembly, AnalysisPriority.DISASSEMBLY);
@@ -894,7 +894,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
setbookmark = options.getBoolean(OPTION_NAME_BOOKMARKS, setbookmark);
}
protected SequenceSearchState initialize(Program program) {
protected BulkPatternSearcher<Pattern> initialize(Program program) {
potentialFuncResult = new AddressSet();
disassemResult = new AddressSet();
@@ -902,15 +902,15 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
postreqFailedResult = new AddressSet();
funcResult = new AddressSet();
if (explicitState != null) {
return explicitState;
if (explicitSearcher != null) {
return explicitSearcher;
}
// TODO: Check the times on the patterns files, maybe reload them!
// could get times of all files and record them to check times.
// filelist keeps getting re-parsed...!
if (rootState != null) {
return rootState;
if (patternSearcher != null) {
return patternSearcher;
}
ArrayList<Pattern> patternlist = new ArrayList<>();
@@ -930,9 +930,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
return null;
}
SequenceSearchState<Pattern> root = SequenceSearchState.buildStateMachine(patternlist);
return root;
BulkPatternSearcher<Pattern> searcher = new BulkPatternSearcher<>(patternlist);
return searcher;
}
private ArrayList<Pattern> readPatterns(ResourceFile[] filelist, Program program) {
@@ -987,7 +986,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
final class PossibleDelayedFunctionCreator extends AnalyzerAdapter {
PossibleDelayedFunctionCreator() {
super(FunctionStartAnalyzer.FUNCTION_START_SEARCH + " delayed", AnalysisPriority.DATA_ANALYSIS.after());
super(FunctionStartAnalyzer.FUNCTION_START_SEARCH + " delayed",
AnalysisPriority.DATA_ANALYSIS.after());
}
@Override
@@ -1001,7 +1001,7 @@ final class PossibleDelayedFunctionCreator extends AnalyzerAdapter {
if (hasConditionalReferences(addedProgram, address)) {
continue;
}
// Check for any function containing the potential start detected earlier in analysis
Function funcAt =
addedProgram.getFunctionManager().getFunctionContaining(address);
@@ -1017,7 +1017,7 @@ final class PossibleDelayedFunctionCreator extends AnalyzerAdapter {
}
functionStarts.add(address);
}
// create functions that still don't exist/overlap
new CreateFunctionCmd(functionStarts, false).applyTo(addedProgram, addedMonitor);
return true;

View File

@@ -1,13 +1,12 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +18,8 @@ package ghidra.app.analyzers;
import ghidra.app.services.AnalysisPriority;
import ghidra.app.services.AnalyzerType;
import ghidra.program.model.listing.Program;
import ghidra.util.bytesearch.SequenceSearchState;
import ghidra.util.bytesearch.BulkPatternSearcher;
import ghidra.util.bytesearch.Pattern;
public class FunctionStartDataPostAnalyzer extends FunctionStartAnalyzer {
protected static final String FUNCTION_START_POST_SEARCH = "Function Start Post Search";
@@ -35,13 +35,13 @@ public class FunctionStartDataPostAnalyzer extends FunctionStartAnalyzer {
if (!super.canAnalyze(program)) {
return false;
}
SequenceSearchState localRoot = initialize(program);
if (localRoot == null) {
BulkPatternSearcher<Pattern> localSearcher = initialize(program);
if (localSearcher == null) {
return false;
}
if (hasDataConstraints) {
// cache the localRoot
rootState = localRoot;
// cache the pattern searcher
patternSearcher = localSearcher;
return true;
}
return false;

View File

@@ -21,7 +21,8 @@ import ghidra.app.util.importer.MessageLog;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.address.AddressSetView;
import ghidra.program.model.listing.Program;
import ghidra.util.bytesearch.SequenceSearchState;
import ghidra.util.bytesearch.BulkPatternSearcher;
import ghidra.util.bytesearch.Pattern;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@@ -48,7 +49,7 @@ public class FunctionStartFuncAnalyzer extends FunctionStartAnalyzer {
if (set.isEmpty()) {
return true;
}
return super.added(program, set, monitor, log);
}
@@ -57,13 +58,13 @@ public class FunctionStartFuncAnalyzer extends FunctionStartAnalyzer {
if (!super.canAnalyze(program)) {
return false;
}
SequenceSearchState localRoot = initialize(program);
if (localRoot == null) {
BulkPatternSearcher<Pattern> localSearcher = initialize(program);
if (localSearcher == null) {
return false;
}
if (hasFunctionStartConstraints) {
// cache the localRoot
rootState = localRoot;
// cache the local pattern searcher
patternSearcher = localSearcher;
return true;
}
return false;

View File

@@ -1,13 +1,12 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +18,8 @@ package ghidra.app.analyzers;
import ghidra.app.services.AnalysisPriority;
import ghidra.app.services.AnalyzerType;
import ghidra.program.model.listing.Program;
import ghidra.util.bytesearch.SequenceSearchState;
import ghidra.util.bytesearch.BulkPatternSearcher;
import ghidra.util.bytesearch.Pattern;
public class FunctionStartPostAnalyzer extends FunctionStartAnalyzer {
protected static final String FUNCTION_START_POST_SEARCH = "Function Start Post Search";
@@ -35,13 +35,13 @@ public class FunctionStartPostAnalyzer extends FunctionStartAnalyzer {
if (!super.canAnalyze(program)) {
return false;
}
SequenceSearchState localRoot = initialize(program);
if (localRoot == null) {
BulkPatternSearcher<Pattern> localSearcher = initialize(program);
if (localSearcher == null) {
return false;
}
if (hasCodeConstraints || hasDataConstraints) {
// cache the localRoot
rootState = localRoot;
// cache the local pattern searcher
patternSearcher = localSearcher;
return true;
}
return false;

View File

@@ -65,10 +65,6 @@ public class ClipboardPanel extends JPanel {
private boolean onlyPrePatterns;
/**
* Class for building the pattern clipboard
* @param plugin
*/
public ClipboardPanel(FunctionBitPatternsExplorerPlugin plugin) {
super();
BoxLayout mainLayout = new BoxLayout(this, BoxLayout.Y_AXIS);
@@ -121,8 +117,8 @@ public class ClipboardPanel extends JPanel {
MatchAction[] actions = getMatchActions(funcStartAnalyzer, pattern);
pattern.setMatchActions(actions);
}
SequenceSearchState<Pattern> root = SequenceSearchState.buildStateMachine(patternList);
funcStartAnalyzer.setExplicitState(root);
BulkPatternSearcher<Pattern> searcher = new BulkPatternSearcher<>(patternList);
funcStartAnalyzer.setExplicitState(searcher);
AutoAnalysisManager autoManager =
AutoAnalysisManager.getAnalysisManager(currentProgram);
autoManager.scheduleOneTimeAnalysis(funcStartAnalyzer,
@@ -214,7 +210,7 @@ public class ClipboardPanel extends JPanel {
Msg.showWarn(this, this, "Only Pre-Patterns",
"Only Pre-Patterns in selection: no true/false positive information will be calculated.");
}
SequenceSearchState<Pattern> root = SequenceSearchState.buildStateMachine(patternList);
BulkPatternSearcher<Pattern> searcher = new BulkPatternSearcher<>(patternList);
indexToSize.clear();
for (Pattern pattern : patternList) {
indexToSize.put(pattern.getIndex(), pattern.getSize());
@@ -230,17 +226,17 @@ public class ClipboardPanel extends JPanel {
if (!block.isExecute()) {
continue;
}
searchBlock(root, block, matchStats, currentProgram, TaskMonitor.DUMMY);
searchBlock(searcher, block, matchStats, currentProgram, TaskMonitor.DUMMY);
}
return matchStats;
}
private void searchBlock(SequenceSearchState<Pattern> root, MemoryBlock block,
private void searchBlock(BulkPatternSearcher<Pattern> searcher, MemoryBlock block,
PatternEvaluationStats matchStats, Program program, TaskMonitor monitor) {
ArrayList<Match<Pattern>> mymatches = new ArrayList<>();
try {
root.apply(block.getData(), mymatches, monitor);
searcher.search(block.getData(), mymatches, monitor);
}
catch (IOException e) {
e.printStackTrace();
@@ -309,6 +305,7 @@ public class ClipboardPanel extends JPanel {
addSeparator(pattern.getHexString(), index), funcStart, postBits, totalBits);
matchStats.addRowObject(rowObject);
}
private int getNumPostBits(Pattern pattern) {
int marked = pattern.getMarkOffset();
if (marked == 0) {
@@ -316,6 +313,7 @@ public class ClipboardPanel extends JPanel {
}
return pattern.getNumFixedBits() - pattern.getNumInitialFixedBits(marked);
}
private PatternMatchType getMatchType(Program program, Address funcStart,
ContextRegisterFilter cRegFilter) {
if (cRegFilter != null) {