From e1e10c27b79d82c7f8c2663928b72c9c65623bdb Mon Sep 17 00:00:00 2001 From: ghidragon <106987263+ghidragon@users.noreply.github.com> Date: Tue, 13 Jan 2026 13:44:39 -0500 Subject: [PATCH] GP-6309 Refactor SequenceSearchState into BulkPatternSearcher --- .../matcher/BulkPatternByteMatcher.java | 53 ++ .../base/memsearch/matcher/ByteMatcher.java | 2 +- .../memsearch/matcher/InvalidByteMatcher.java | 2 +- .../MaskedByteSequenceByteMatcher.java | 4 +- .../memsearch/matcher/RegExByteMatcher.java | 2 +- .../matcher/UserInputByteMatcher.java | 2 +- .../memsearch/searcher/MemorySearcher.java | 3 +- .../bytesearch}/AddressableByteSequence.java | 2 +- .../util/bytesearch/BulkPatternSearcher.java | 464 +++++++++++++++++ .../ghidra/util/bytesearch/BytePattern.java | 38 ++ .../bytesearch}/ByteSequence.java | 2 +- .../util/bytesearch/DittedBitSequence.java | 4 +- .../bytesearch}/ExtendedByteSequence.java | 2 +- .../InputStreamBufferByteSequence.java | 71 +++ .../bytesearch/MemoryBytePatternSearcher.java | 27 +- .../util/bytesearch/SequenceSearchState.java | 468 ------------------ .../bytesequence/ByteArrayByteSequence.java | 2 + .../ByteArrayByteSequenceTest.java | 3 + .../MaskedBytesSequenceByteMatcherTest.java | 2 +- .../bytesequence/RegExByteMatcherTest.java | 2 +- .../bytesearch/BulkPatternSearcherTest.java | 273 ++++++++++ .../ghidra_scripts/DumpMissedStarts.java | 31 +- .../ghidra_scripts/PatternStats.java | 7 +- .../app/analyzers/FunctionStartAnalyzer.java | 184 +++---- .../FunctionStartDataPostAnalyzer.java | 16 +- .../analyzers/FunctionStartFuncAnalyzer.java | 13 +- .../analyzers/FunctionStartPostAnalyzer.java | 16 +- .../bitpatterns/gui/ClipboardPanel.java | 18 +- .../analysis/AARCH64PltThunkAnalyzer.java | 140 +++--- 29 files changed, 1156 insertions(+), 697 deletions(-) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java rename Ghidra/Features/Base/src/main/java/ghidra/{features/base/memsearch/bytesequence => util/bytesearch}/AddressableByteSequence.java (98%) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java rename Ghidra/Features/Base/src/main/java/ghidra/{features/base/memsearch/bytesequence => util/bytesearch}/ByteSequence.java (97%) rename Ghidra/Features/Base/src/main/java/ghidra/{features/base/memsearch/bytesequence => util/bytesearch}/ExtendedByteSequence.java (98%) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/InputStreamBufferByteSequence.java delete mode 100644 Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java create mode 100644 Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java new file mode 100644 index 0000000000..7d53f1f4fa --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java @@ -0,0 +1,53 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.features.base.memsearch.matcher; + +import java.util.ArrayList; +import java.util.List; + +import ghidra.util.bytesearch.*; + +/** + * {@link ByteMatcher} that uses a BulkPatternSearcher to simultaneously search for multiple + * byte patterns. + * + * @param the specific pattern type + */ +public class BulkPatternByteMatcher implements ByteMatcher { + + private BulkPatternSearcher matcher; + + /** + * Constructor + * @param patterns the list of patterns that this byte matcher will simultaneously search for + */ + public BulkPatternByteMatcher(List patterns) { + matcher = new BulkPatternSearcher(patterns); + } + + @Override + public Iterable> match(ExtendedByteSequence bytes) { + List> matches = new ArrayList<>(); + matcher.search(bytes, matches, 0); + return matches; + } + + @Override + public String getDescription() { + return "Bulk Pattern Searcher"; + } + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/ByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/ByteMatcher.java index 47467fb610..526809dddc 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/ByteMatcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/ByteMatcher.java @@ -15,7 +15,7 @@ */ package ghidra.features.base.memsearch.matcher; -import ghidra.features.base.memsearch.bytesequence.ExtendedByteSequence; +import ghidra.util.bytesearch.ExtendedByteSequence; import ghidra.util.bytesearch.Match; /** diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/InvalidByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/InvalidByteMatcher.java index ac9d86cd7b..f798d5eaaf 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/InvalidByteMatcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/InvalidByteMatcher.java @@ -17,8 +17,8 @@ package ghidra.features.base.memsearch.matcher; import org.apache.commons.collections4.iterators.EmptyIterator; -import ghidra.features.base.memsearch.bytesequence.ExtendedByteSequence; import ghidra.features.base.memsearch.format.SearchFormat; +import ghidra.util.bytesearch.ExtendedByteSequence; import ghidra.util.bytesearch.Match; import util.CollectionUtils; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/MaskedByteSequenceByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/MaskedByteSequenceByteMatcher.java index 65f50ee8d0..a5291977f6 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/MaskedByteSequenceByteMatcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/MaskedByteSequenceByteMatcher.java @@ -19,10 +19,8 @@ import java.util.Iterator; import org.bouncycastle.util.Arrays; -import ghidra.features.base.memsearch.bytesequence.ByteSequence; -import ghidra.features.base.memsearch.bytesequence.ExtendedByteSequence; import ghidra.features.base.memsearch.gui.SearchSettings; -import ghidra.util.bytesearch.Match; +import ghidra.util.bytesearch.*; /** * {@link ByteMatcher} where the user search input has been parsed into a sequence of bytes and diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/RegExByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/RegExByteMatcher.java index 2aa9841200..a7f9beff02 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/RegExByteMatcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/RegExByteMatcher.java @@ -21,8 +21,8 @@ import java.util.regex.Pattern; import javax.help.UnsupportedOperationException; -import ghidra.features.base.memsearch.bytesequence.ExtendedByteSequence; import ghidra.features.base.memsearch.gui.SearchSettings; +import ghidra.util.bytesearch.ExtendedByteSequence; import ghidra.util.bytesearch.Match; /** diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/UserInputByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/UserInputByteMatcher.java index 5a2caf8d0f..4386fd3e8e 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/UserInputByteMatcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/UserInputByteMatcher.java @@ -17,8 +17,8 @@ package ghidra.features.base.memsearch.matcher; import java.util.Objects; -import ghidra.features.base.memsearch.bytesequence.ExtendedByteSequence; import ghidra.features.base.memsearch.gui.SearchSettings; +import ghidra.util.bytesearch.ExtendedByteSequence; import ghidra.util.bytesearch.Match; public abstract class UserInputByteMatcher implements ByteMatcher { diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/searcher/MemorySearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/searcher/MemorySearcher.java index 92888aaf62..b49ab947d7 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/searcher/MemorySearcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/searcher/MemorySearcher.java @@ -17,11 +17,10 @@ package ghidra.features.base.memsearch.searcher; import java.util.function.Predicate; -import ghidra.features.base.memsearch.bytesequence.*; import ghidra.features.base.memsearch.bytesource.AddressableByteSource; import ghidra.features.base.memsearch.matcher.ByteMatcher; import ghidra.program.model.address.*; -import ghidra.util.bytesearch.Match; +import ghidra.util.bytesearch.*; import ghidra.util.datastruct.Accumulator; import ghidra.util.task.TaskMonitor; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/AddressableByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java similarity index 98% rename from Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/AddressableByteSequence.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java index 28987810eb..ddf28278af 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/AddressableByteSequence.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ghidra.features.base.memsearch.bytesequence; +package ghidra.util.bytesearch; import ghidra.features.base.memsearch.bytesource.AddressableByteSource; import ghidra.program.model.address.Address; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java new file mode 100644 index 0000000000..3ea2704f7b --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java @@ -0,0 +1,464 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Array; +import java.util.*; + +import ghidra.util.task.TaskMonitor; + +/** + * State machine for searching for a list of {@link BytePattern}s simultaneously in a byte + * sequence. Once this BulkPatternMatcher is constructed from a list of patterns, it can + * be used any number of times to search byte sequences. There are an assortment of search methods + * to meet various client needs. + *

+ * The search methods break down into the following categories: + * 1) Searching a byte buffer with the result being an iterator over matches. + * 2) Searching a byte buffer with the results being added to a given list. + * 3) Searching an input stream with the results being added to a given list. + * 4) Searching an {@link ExtendedByteSequence} with the results being added to a given list + *

+ * In addition, the byte buffer methods all have a variation that takes an additional parameter + * stating how many of the bytes in the buffer are searchable. (The buffer is not full). + * Also, the input stream method has a variation where the max bytes to read from the stream + * is given. + *

+ * @param The specific pattern type + */ +public class BulkPatternSearcher { + private static final int DEFAULT_BUFFER_SIZE = 4096; + + private List patterns; + private SearchState startState; + private int bufferSize = DEFAULT_BUFFER_SIZE; + private int uniqueStateCount; + + /** + * Constructor + * @param patterns the list of patterns that can be search simultaneously using an internal + * finite state machine + */ + public BulkPatternSearcher(List patterns) { + this.patterns = patterns; + startState = buildStateMachine(); + } + + /** + * Search the given byte buffer for any of this searcher's patterns. + * @param input the byte buffer to search + * @return An iterator that will return pattern matches one at a time. + */ + public Iterator> search(byte[] input) { + return new ByteArrayMatchIterator(input); + } + + /** + * Search the given byte buffer up the specified length for any of this searcher's patterns. + * @param input the byte buffer to search + * @param length the actual number of the bytes in the buffer to search. + * @return An iterator that will return pattern matches one at a time. + */ + public Iterator> search(byte[] input, int length) { + return new ByteArrayMatchIterator(input, length); + } + + /** + * Searches for the patterns in the given byte array, adding match results to the given list + * of results. + * @param input the byte array to search for patterns + * @param results the list of match results to populate + */ + public void search(byte[] input, List> results) { + search(input, input.length, results); + } + + /** + * Searches for the patterns in the given byte array, adding match results to the given list + * of results. + * @param input the byte array to search for patterns + * @param numBytes the number of valid bytes in the input buffer to search + * @param results the list of match results to populate + */ + public void search(byte[] input, int numBytes, List> results) { + for (int patternStart = 0; patternStart < numBytes; patternStart++) { + SearchState state = startState; + for (int i = patternStart; i < numBytes; i++) { + int index = input[i] & 0xff; // turn byte value into unsigned int (0-255) + SearchState nextState = state.nextStates[index]; + if (nextState == null) { + break; + } + nextState.addMatchesForCompletedPatterns(results, patternStart); + state = nextState; + } + } + } + + /** + * Searches for the patterns in the given byte array that start at the first byte in the array. + * Resulting matches are added to the given results list. + * @param input the byte array to search for patterns + * @param numBytes the number of bytes to use from the given byte array. (The byte array might + * not be fully populated with valid data.) + * @param results the list of match results to populate + */ + public void matches(byte[] input, int numBytes, List> results) { + SearchState state = startState; + for (int i = 0; i < numBytes; i++) { + int index = input[i] & 0xff; // turn byte value into unsigned int (0-255) + SearchState nextState = state.nextStates[index]; + if (nextState == null) { + break; + } + nextState.addMatchesForCompletedPatterns(results, 0); + state = nextState; + } + } + + /** + * Searches for the patterns in the given {@link ExtendedByteSequence}, adding match results + * to the given list of results. + * @param bytes the extended byte sequence to search + * @param results the list of match results to populate + * @param chunkOffset a constant offset to add to the pattern starts found in this buffer. + * Users of this method may have split a larger byte sequence into chunks and the final match + * position needs to be the sum of the chunk offset plus the offset within this chunk. + */ + public void search(ExtendedByteSequence bytes, List> results, int chunkOffset) { + for (int patternStart = 0; patternStart < bytes.getLength(); ++patternStart) { + SearchState state = startState; + for (int j = patternStart; j < bytes.getExtendedLength(); j++) { + int index = bytes.getByte(j) & 0xff; + SearchState nextState = state.nextStates[index]; + if (nextState == null) { + break; + } + nextState.addMatchesForCompletedPatterns(results, patternStart + chunkOffset); + state = nextState; + } + } + } + + /** + * Searches for the patterns in the given input stream, adding match results to the given list + * of results. + * @param is the input stream of bytes to scan for patterns + * @param results the list of match results to populate + * @param monitor the task monitor + * @throws IOException if an exception occurs reading the input stream + */ + public void search(InputStream is, List> results, TaskMonitor monitor) + throws IOException { + search(is, -1, results, monitor); + } + + /** + * Searches for the patterns in the given input stream, adding match results to the given + * list of results. + *

+ * @param inputStream the input stream of bytes to scan for patterns + * @param maxRead the maximum offset into the input stream where a match can start. Additional + * bytes can be read from the stream to complete patterns + * @param results the list of match results to populate + * @param monitor the task monitor + * @throws IOException if an exception occurs reading the input stream + */ + public void search(InputStream inputStream, long maxRead, List> results, + TaskMonitor monitor) throws IOException { + RestrictedStream restrictedStream = new RestrictedStream(inputStream, maxRead); + int maxPatternLength = getLongestPatternLength(); + int bufSize = Math.max(maxPatternLength, bufferSize); + int offset = 0; + + // The basic strategy is to use two byte buffers and create a virtual buffer with those two + // buffers. The first pass will look for patterns that start in the 1st buffer but can + // extend into the second buffer. This is to ensure that we find patterns that span + // buffers. + // + // Then the second buffer is swapped to be the 1st buffer and new data is read + // into the what was the 1st buffer, but is now the 2nd buffer. This pattern is repeated + // until all the data is processed up to the number of bytes specified by the maxRead. No + // patterns will matched in any data in the stream past that point, but data past that point + // may be used to complete a pattern. + + InputStreamBufferByteSequence bufA = new InputStreamBufferByteSequence(bufSize); + InputStreamBufferByteSequence bufB = new InputStreamBufferByteSequence(bufSize); + InputStreamBufferByteSequence buf1 = bufA; + InputStreamBufferByteSequence buf2 = bufB; + buf1.load(restrictedStream, bufSize); + buf2.load(restrictedStream, bufSize); + + while (buf1.getLength() > 0 && buf2.getLength() > 0) { + if (monitor.isCancelled()) { + return; + } + + ExtendedByteSequence combined = new ExtendedByteSequence(buf1, buf2, maxPatternLength); + search(combined, results, offset); + monitor.incrementProgress(buf1.getLength()); + offset += buf1.getLength(); + + // swap buffers and load data into second buffer + buf1 = buf2; + buf2 = buf1 == bufA ? bufB : bufA; + buf2.load(restrictedStream, bufSize); + } + // just have to read a bit more to finish last pattern and we go beyond restricted maxRead + // so use unrestricted stream + buf2.load(inputStream, maxPatternLength); + ExtendedByteSequence combined = new ExtendedByteSequence(buf1, buf2, maxPatternLength); + search(combined, results, offset); + monitor.incrementProgress(buf1.getLength()); + } + + /** + * Sets the buffer size used when using one of the search methods that takes an input stream. + * Mostly used for testing. + * @param bufferSize the size of the buffers to use when searching input streams. + */ + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + private SearchState buildStateMachine() { + Queue> unprocessed = new ArrayDeque<>(); + + // We use a map that uses the same unique state as the key and the value. + // This way, if you create a new state that is equal to an existing state in the cache, + // you can use the new state instance as a key to get the existing equivalent state. + Map, SearchState> dedupCache = new HashMap<>(); + SearchState start = new SearchState(patterns, 0); + unprocessed.add(start); + while (!unprocessed.isEmpty()) { + SearchState next = unprocessed.remove(); + next.computeTransitions(unprocessed, dedupCache); + } + uniqueStateCount = dedupCache.size() + 1; // add 1 for the root state which wasn't cached + dedupCache.clear(); + return start; + } + + private int getLongestPatternLength() { + int maxLength = 0; + for (T t : patterns) { + maxLength = Math.max(maxLength, t.getSize()); + } + return maxLength; + } + + /** + * {@return the number of unique states generated. Used for testing.} + */ + public int getUniqueStateCount() { + return uniqueStateCount; + } + + private class ByteArrayMatchIterator implements Iterator> { + private byte[] bytes; + private int length; + private int patternStart = 0; + private Queue> resultBuffer = new ArrayDeque<>(); + + ByteArrayMatchIterator(byte[] input) { + this(input, input.length); + } + + ByteArrayMatchIterator(byte[] input, int length) { + this.bytes = input; + this.length = Math.min(length, bytes.length); + findNext(); + } + + private void findNext() { + while (patternStart < length && resultBuffer.isEmpty()) { + SearchState state = startState; + for (int i = patternStart; i < length; i++) { + int index = bytes[i] & 0xff; // turn byte value into unsigned int (0-255) + state = state.nextStates[index]; + if (state == null) { + break; + } + state.addMatchesForCompletedPatterns(resultBuffer, patternStart); + } + patternStart++; + } + } + + @Override + public boolean hasNext() { + return !resultBuffer.isEmpty(); + } + + @Override + public Match next() { + Match nextResult = resultBuffer.poll(); + if (resultBuffer.isEmpty()) { + findNext(); + } + + return nextResult; + } + } + + /** + * A single state in the state machine that represents one or more active patterns that have + * matched the sequence of bytes so far. + * + * @param the specific type of patterns being search for + */ + private static class SearchState { + private List activePatterns; // patterns that have matched the input bytes so far + private List completedPatterns; // the active patterns that have completely matched + private SearchState[] nextStates; // next state (transition) for each possible input byte + private int level; // the number of bytes that have been matched so far + private int hash; + + SearchState(List activePatterns, int level) { + this.activePatterns = activePatterns; + this.level = level; + hash = Objects.hash(activePatterns, level); + } + + void computeTransitions(Queue> unresolved, + Map, SearchState> cache) { + completedPatterns = buildFullyMatchedPatternsList(); + nextStates = createTransitionArray(); + if (completedPatterns != null && completedPatterns.size() == activePatterns.size()) { + return; // we are a terminal state + } + for (int inputValue = 0; inputValue < 256; inputValue++) { + List matchedPatterns = getMatchedPatterns(inputValue); + if (!matchedPatterns.isEmpty()) { + nextStates[inputValue] = getSearchState(matchedPatterns, cache, unresolved); + } + } + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + SearchState other = (SearchState) obj; + if (hash != other.hash) { + return false; + } + return level == other.level && Objects.equals(activePatterns, other.activePatterns); + } + + @SuppressWarnings("unchecked") + private SearchState[] createTransitionArray() { + return (SearchState[]) Array.newInstance(getClass(), 256); + } + + private SearchState getSearchState(List patterns, + Map, SearchState> cache, Queue> unresolved) { + SearchState newState = new SearchState(patterns, level + 1); + SearchState existing = cache.get(newState); + if (existing != null) { + return existing; + } + cache.put(newState, newState); + unresolved.add(newState); + return newState; + } + + private List getMatchedPatterns(int inputValue) { + List matchedPatterns = new ArrayList<>(); + for (T pattern : activePatterns) { + if (pattern.isMatch(level, inputValue)) { + matchedPatterns.add(pattern); + } + } + return matchedPatterns; + } + + private void addMatchesForCompletedPatterns(Collection> results, int i) { + if (completedPatterns == null) { + return; + } + for (T pattern : completedPatterns) { + results.add(new Match(pattern, i, pattern.getSize())); + } + } + + private List buildFullyMatchedPatternsList() { + List list = new ArrayList<>(); + for (T pattern : activePatterns) { + if (pattern.getSize() == level) { + list.add(pattern); + } + } + return list.isEmpty() ? null : list; + } + } + + private static class RestrictedStream extends InputStream { + private long maxRead; + private long totalRead; + private InputStream is; + + RestrictedStream(InputStream is, long maxRead) { + this.is = is; + this.maxRead = maxRead; + } + + @Override + public int read(byte[] buf) throws IOException { + return read(buf, 0, buf.length); + } + + @Override + public int read(byte[] buf, int offset, int amount) throws IOException { + int amountToRead = amount; + if (maxRead >= 0) { + long remaining = maxRead - totalRead; + amountToRead = (int) Math.min(remaining, amount); + } + int n = is.read(buf, offset, amountToRead); + n = n > 0 ? n : 0; + totalRead += n; + return n; + } + + @Override + public int read() throws IOException { + if (totalRead >= maxRead) { + return -1; + } + int value = is.read(); + if (value < 0) { + return -1; + } + totalRead++; + return value; + } + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java new file mode 100644 index 0000000000..563740cb61 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java @@ -0,0 +1,38 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +/** + * Interface for Patterns that can be combined into a single state machine that can be + * simultaneously searched for in a byte sequence. + */ +public interface BytePattern { + + /** + * {@return the size of this pattern.} + */ + public int getSize(); + + /** + * Checks if this pattern matches a byte value at a specific offset into the pattern. + * @param patternOffset the position in the pattern to check if it matches the given byte value + * @param byteValue the byte value to check if it matches the pattern at the given offset. This + * value is passed as an int so that the byte can be treated as unsigned. + * @return true if this pattern matches the given byte value at the given pattern offset. + */ + public boolean isMatch(int patternOffset, int byteValue); + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ByteSequence.java similarity index 97% rename from Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ByteSequence.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ByteSequence.java index a4818d754a..eee435b818 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ByteSequence.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ByteSequence.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ghidra.features.base.memsearch.bytesequence; +package ghidra.util.bytesearch; /** * An interface for accessing bytes from a byte source. diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java index 8ade3fdec4..f7886a2d71 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java @@ -34,7 +34,7 @@ import ghidra.xml.XmlPullParser; * where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary) */ -public class DittedBitSequence { +public class DittedBitSequence implements BytePattern { //Given a byte 0-255 (NOT a signed byte), retrieves its popcount. public static int[] popcount = { 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, //0-15 @@ -214,6 +214,7 @@ public class DittedBitSequence { * * @return true if the byte matches the sequence mask/value */ + @Override public boolean isMatch(int pos, int val) { if (pos >= bits.length) { return false; @@ -244,6 +245,7 @@ public class DittedBitSequence { * * @return size in bytes */ + @Override public int getSize() { return bits.length; } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ExtendedByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java similarity index 98% rename from Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ExtendedByteSequence.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java index 46d324751e..dc265e579e 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/bytesequence/ExtendedByteSequence.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ghidra.features.base.memsearch.bytesequence; +package ghidra.util.bytesearch; /** * A class for accessing a contiguous sequence of bytes from some underlying byte source to diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/InputStreamBufferByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/InputStreamBufferByteSequence.java new file mode 100644 index 0000000000..62ff1228a0 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/InputStreamBufferByteSequence.java @@ -0,0 +1,71 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +import java.io.IOException; +import java.io.InputStream; + +/** + * ByteSequence that buffers an {@link InputStream} + */ +public class InputStreamBufferByteSequence implements ByteSequence { + private byte[] bytes; + private int validDataLength; + + public InputStreamBufferByteSequence(int bufferSize) { + bytes = new byte[bufferSize]; + } + + /** + * Loads data into this byte sequence from the given input stream + * @param is the input stream to read bytes from + * @param amount the number of bytes to read from the stream + * @throws IOException if an error occurs reading from the stream + */ + public void load(InputStream is, int amount) throws IOException { + if (amount > bytes.length) { + throw new IllegalArgumentException("Attempted to read greater that buffer size!"); + } + int numRead = is.read(bytes, 0, amount); + validDataLength = numRead >= 0 ? numRead : 0; + } + + @Override + public int getLength() { + return validDataLength; + } + + @Override + public byte getByte(int index) { + return bytes[index]; + } + + @Override + public boolean hasAvailableBytes(int index, int length) { + return index >= 0 && index + length <= validDataLength; + } + + @Override + public byte[] getBytes(int index, int length) { + if (index < 0 || index + length > validDataLength) { + throw new IndexOutOfBoundsException(); + } + byte[] results = new byte[length]; + System.arraycopy(bytes, index, results, 0, length); + return results; + } + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java index 47da2c19a3..6dd0ca4d34 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java @@ -36,7 +36,7 @@ import ghidra.util.task.TaskMonitor; public class MemoryBytePatternSearcher { private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32; - SequenceSearchState root = null; + BulkPatternSearcher patternSearcher = null; ArrayList patternList; @@ -58,13 +58,13 @@ public class MemoryBytePatternSearcher { } /** - * Create with an initialized SequenceSearchState + * Create with an initialized BulkPatternSearcher * @param searchName name of search - * @param root search state pre-initialized + * @param searcher search state pre-initialized */ - public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) { + public MemoryBytePatternSearcher(String searchName, BulkPatternSearcher searcher) { this.searchName = searchName; - this.root = root; + this.patternSearcher = searcher; } /** @@ -101,8 +101,8 @@ public class MemoryBytePatternSearcher { */ public void search(Program program, AddressSetView searchSet, TaskMonitor monitor) throws CancelledException { - if (root == null) { - root = SequenceSearchState.buildStateMachine(patternList); + if (patternSearcher == null) { + patternSearcher = new BulkPatternSearcher<>(patternList); } numToSearch = getNumToSearch(program, searchSet); @@ -125,7 +125,7 @@ public class MemoryBytePatternSearcher { } try { - searchBlock(root, program, block, searchSet, monitor); + searchBlock(patternSearcher, program, block, searchSet, monitor); } catch (IOException e) { Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName); @@ -155,7 +155,7 @@ public class MemoryBytePatternSearcher { } /** - * Search through bytes of a memory block using the finite state machine -root- + * Search through bytes of a memory block using the finite state machine (BulkPatterMatcher) * Apply any additional rules for matching patterns. * * @param program is the Program being searched @@ -164,8 +164,8 @@ public class MemoryBytePatternSearcher { * @throws IOException exception during read of memory * @throws CancelledException canceled search */ - private void searchBlock(SequenceSearchState rootState, Program program, MemoryBlock block, - AddressSetView restrictSet, TaskMonitor monitor) + private void searchBlock(BulkPatternSearcher searcher, Program program, + MemoryBlock block, AddressSetView restrictSet, TaskMonitor monitor) throws IOException, CancelledException { // if no restricted set, make restrict set the full block @@ -216,7 +216,7 @@ public class MemoryBytePatternSearcher { InputStream data = block.getData(); data.skip(blockOffset); - rootState.apply(data, maxBlockSearchLength, mymatches, monitor); + searcher.search(data, maxBlockSearchLength, mymatches, monitor); monitor.checkCancelled(); monitor.setMessage(searchName + " (Examine Matches)"); @@ -230,7 +230,8 @@ public class MemoryBytePatternSearcher { matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size()))); Match match = mymatches.get(i); Pattern pattern = match.getPattern(); - Address addr = blockStartAddr.add(pattern.getMarkOffset()+match.getStart() + blockOffset); + Address addr = + blockStartAddr.add(pattern.getMarkOffset() + match.getStart() + blockOffset); long totalOffset = streamoffset + blockOffset + match.getStart(); if (!pattern.checkPostRules(totalOffset)) { continue; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java deleted file mode 100644 index c3df31db86..0000000000 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java +++ /dev/null @@ -1,468 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ghidra.util.bytesearch; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Array; -import java.util.*; - -import ghidra.util.task.TaskMonitor; - -/** - * SeqenceSearchState holds the state of a search for a DittedBitSequence within a byte - * sequence. - */ -public class SequenceSearchState implements Comparable> { - - private static final int PATTERN_ENDED = Integer.MAX_VALUE; - private SequenceSearchState parent; - private ArrayList possible; // Patterns that could still match in this state - private ArrayList success; // Patterns that have matched successfully if we reached this state - private SequenceSearchState[] trans; // State transitions based on next byte - - /** - * Construct a sub sequence state with a parent sequence - * - * @param parent parent SequenceSearchState - */ - public SequenceSearchState(SequenceSearchState parent) { - this.parent = parent; - possible = new ArrayList(); - success = null; - trans = null; - } - - /** - * @return maximum number of bytes that could be matched by this sequence - */ - public int getMaxSequenceSize() { - int max = 0; - for (DittedBitSequence element : possible) { - int val = element.getSize(); - if (val > max) { - max = val; - } - } - return max; - } - - /** - * Add a pattern to this search sequence. The last pattern added is the successful - * match pattern. - * @param pat pattern to add - * @param pos position within the current set of patterns to add this pattern - */ - public void addSequence(T pat, int pos) { - possible.add(pat); - if (pos == pat.getSize()) { - if (success == null) { - success = new ArrayList(); - } - success.add(pat); - } - } - - /** - * Sort the sequences that have been added - */ - public void sortSequences() { - Comparator comp = new Comparator() { - @Override - public int compare(DittedBitSequence o1, DittedBitSequence o2) { - return o1.getIndex() - o2.getIndex(); - } - }; - Collections.sort(possible, comp); - if (success != null) { - Collections.sort(success, comp); - } - } - - @Override - public int compareTo(SequenceSearchState o) { - int i = 0; - for (;;) { - if (possible.size() <= i) { - if (o.possible.size() <= i) { - return 0; - } - return -1; - } - if (o.possible.size() <= i) { - return 1; - } - int indus = possible.get(i).getIndex(); // Lexicographic compare an sequence of sequences - int indthem = o.possible.get(i).getIndex(); - if (indus != indthem) { - return (indus < indthem) ? -1 : 1; - } - i += 1; - } - } - - private void buildSingleTransition(ArrayList> all, int pos, int val) { - SequenceSearchState newstate = null; - for (T curpat : possible) { - if (curpat.isMatch(pos, val)) { - if (newstate == null) { - newstate = new SequenceSearchState<>(this); - } - newstate.addSequence(curpat, pos + 1); - } - } - trans[val] = newstate; - if (newstate != null) { - newstate.sortSequences(); - all.add(newstate); - } - } - - private void exportSuccess(ArrayList> match, int offset) { - for (T succes : success) { // If we found matches - Match newmatch = new Match(succes, offset, succes.getSize()); - match.add(newmatch); - } - } - - /** - * Merge in -op- and this as a single state - * @param op - */ - private void merge(SequenceSearchState op) { - SequenceSearchState parent = op.parent; - for (int i = 0; i < 256; ++i) { - if (parent.trans[i] == op) { - parent.trans[i] = this; // Should be replaced with this - } - } - if (op.success != null) { // Merge - if (success == null) { - success = op.success; - } - else { - ArrayList tmp = new ArrayList<>(); - int i = 0; - int j = 0; - int curpat = -1; - int thispat = success.get(i).getIndex(); - int oppat = op.success.get(j).getIndex(); - while ((i < success.size()) || (j < op.success.size())) { - if (thispat == oppat) { - if (curpat != thispat) { - tmp.add(success.get(i)); - curpat = thispat; - } - i += 1; - j += 1; - thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex(); - oppat = - (j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex(); - } - else if (thispat < oppat) { - if (curpat != thispat) { - tmp.add(success.get(i)); - curpat = thispat; - } - i += 1; - thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex(); - } - else { - if (curpat != oppat) { - tmp.add(op.success.get(j)); - curpat = oppat; - } - j += 1; - oppat = - (j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex(); - } - } - success = tmp; - } - } - } - - /** - * Try to match this Sequence to the byteArray, and add any matches to the match list - * @param bytearray array of bytes to match - * @param numbytes retrict number of bytes to allow to match - * @param match list of matches, the result - */ - public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList> match) { - int subindex = 0; - SequenceSearchState curstate = this; - - do { - if (curstate.success != null) { - curstate.exportSuccess(match, 0); - } - if (subindex >= numbytes) { - return; - } - curstate = curstate.trans[0xff & bytearray[subindex]]; // Perform state transition based on next byte in buffer - subindex += 1; - } - while (curstate != null); - - } - - /** - * Search for patterns in a byte array. All matches are returned. - * @param buffer is the array of bytes to search - * @param match is populated with a Match object for each pattern and position that matches - */ - public void apply(byte[] buffer, ArrayList> match) { - SequenceSearchState curstate; - int subindex; - for (int offset = 0; offset < buffer.length; ++offset) { - curstate = this; // New starting offset -> Root state - subindex = offset; - do { - if (curstate.success != null) { - curstate.exportSuccess(match, offset); - } - if (subindex >= buffer.length) { // if we've run out of bytes, must restart at next offset - break; - } - curstate = curstate.trans[0xff & buffer[subindex]]; // Perform state transition based on next byte - subindex += 1; - } - while (curstate != null); - } - } - - /** - * Search for pattern in the stream -in-. - * @param in - The stream to scan for matches - * @param match - Any matches are appended as Match records to this ArrayList - * @param monitor - if non-null, check for user cancel, and maintain progress info - * @throws IOException - */ - public void apply(InputStream in, ArrayList> match, TaskMonitor monitor) - throws IOException { - apply(in, -1L, match, monitor); - } - - /** - * Search for pattern in the stream -in-. - * @param in - The stream to scan for matches - * @param maxBytes - The maximum number of bytes to scan forward in this stream - * @param match - Any matches are appended as Match records to this ArrayList - * @param monitor - if non-null, check for user cancel, and maintain progress info - * @throws IOException - */ - public void apply(InputStream in, long maxBytes, ArrayList> match, TaskMonitor monitor) - throws IOException { - long progress = monitor.getProgress(); - - int maxSize = getMaxSequenceSize() + 1; - if (maxSize < 4096) { - maxSize = 4096; - } - if (maxBytes > 0) { - maxBytes += getMaxSequenceSize() + 1; - } - byte[] firstBuf = new byte[maxSize]; - byte[] secondBuf = new byte[maxSize]; - byte[] curBuf; - SequenceSearchState curState; - int fullBuffers; // Number of buffers that are completely full - int ra = in.read(firstBuf); - if (ra == firstBuf.length) { - ra = in.read(secondBuf); - if (ra == secondBuf.length) { - fullBuffers = 2; - } - else { - if (ra < 0) { - ra = 0; - } - fullBuffers = 1; - byte[] tmp = new byte[ra]; - for (int i = 0; i < ra; ++i) { - tmp[i] = secondBuf[i]; - } - secondBuf = tmp; - } - } - else if (ra < 0) { - return; // No bytes at all were read - } - else { - byte[] tmp = new byte[ra]; - for (int i = 0; i < ra; ++i) { - tmp[i] = firstBuf[i]; - } - firstBuf = tmp; - fullBuffers = 0; - secondBuf = new byte[0]; - } - int offset = 0; - int bufRelativeOffset = 0; - int subIndex; - while (fullBuffers == 2) { - curState = this; // New starting offset -> Root state - subIndex = bufRelativeOffset; - curBuf = firstBuf; - do { - if (curState.success != null) { - curState.exportSuccess(match, offset); - } - if (subIndex >= curBuf.length) { // check that we have enough bytes in current buffer - curBuf = secondBuf; // If not, switch to secondary buffer - subIndex = 0; - } - curState = curState.trans[0xff & curBuf[subIndex]]; // Perform state transition based on next byte in buffer - subIndex += 1; - } - while (curState != null); - offset += 1; // Advance to next starting offset - if (maxBytes > 0 && offset > maxBytes) { - break; - } - bufRelativeOffset += 1; - if (bufRelativeOffset == firstBuf.length) { // If starting offset no longer falls in firstbuf - byte[] tmp = firstBuf; // Switch firstbuf with secondbuf - firstBuf = secondBuf; - secondBuf = tmp; - ra = in.read(secondBuf); // refill secondbuf (old firstbuf) with new bytes - if (monitor != null) { - if (monitor.isCancelled()) { - return; - } - monitor.setProgress(progress + offset); - } - if (ra != secondBuf.length) { - fullBuffers = 1; - if (ra < 0) { - ra = 0; - } - tmp = new byte[ra]; - for (int i = 0; i < ra; ++i) { - tmp[i] = secondBuf[i]; - } - secondBuf = tmp; - } - bufRelativeOffset = 0; - } - } - - while (fullBuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) { - if (secondBuf.length == 0) { - fullBuffers = 0; - } - curState = this; - subIndex = bufRelativeOffset; - curBuf = firstBuf; - do { - if (curState.success != null) { - curState.exportSuccess(match, offset); - } - if (subIndex >= curBuf.length) { - if (curBuf == secondBuf) { - break; // Out of data, all pending patterns fail - } - curBuf = secondBuf; - subIndex = 0; - if (curBuf.length == 0) { - break; - } - } - curState = curState.trans[0xff & curBuf[subIndex]]; - subIndex += 1; - } - while (curState != null); - offset += 1; - bufRelativeOffset += 1; - if (bufRelativeOffset == firstBuf.length) { - if (fullBuffers == 0) { - break; - } - firstBuf = secondBuf; - fullBuffers = 0; - bufRelativeOffset = 0; - secondBuf = new byte[0]; - } - } - } - - /** - * Build a new transition level for the state machine - * - * @param prev previous search sequences - * @param pos position within the search sequence state for this level - * @return list of possible new search states to be added to the state machine - */ - @SuppressWarnings("unchecked") - static ArrayList> buildTransitionLevel(ArrayList> prev, - int pos) { - ArrayList> res = new ArrayList>(); - Iterator> iterator = prev.iterator(); - while (iterator.hasNext()) { // For each current state - SequenceSearchState next = iterator.next(); - next.trans = (SequenceSearchState[]) Array.newInstance(next.getClass(), 256); - for (int i = 0; i < 256; ++i) { // Try every byte transition - next.buildSingleTransition(res, pos, i); - } - } - if (res.isEmpty()) { - return res; - } - // Prepare to dedup the states - Collections.sort(res); - ArrayList> finalres = new ArrayList>(); - Iterator> iter = res.iterator(); - SequenceSearchState curpat = iter.next(); - finalres.add(curpat); - while (iter.hasNext()) { - SequenceSearchState nextpat = iter.next(); - int comp = curpat.compareTo(nextpat); - if (comp == 0) { // Identical states - curpat.merge(nextpat); - } - else { - curpat = nextpat; - finalres.add(curpat); - } - } - return finalres; - } - - /** - * Build a search state machine from a list of DittedBitSequences - * @param patterns bit sequence patterns - * @return search state the will match the given sequences - */ - static public SequenceSearchState buildStateMachine( - ArrayList patterns) { - SequenceSearchState root = new SequenceSearchState<>(null); - for (int i = 0; i < patterns.size(); ++i) { - T pat = patterns.get(i); - pat.setIndex(i); - root.addSequence(pat, 0); - } - root.sortSequences(); - ArrayList> statelevel = new ArrayList>(); - statelevel.add(root); - int level = 0; - do { - statelevel = buildTransitionLevel(statelevel, level); - level += 1; - } - while (!statelevel.isEmpty()); - return root; - } -} diff --git a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java index e04bc5af86..6dcd80b118 100644 --- a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java +++ b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java @@ -15,6 +15,8 @@ */ package ghidra.features.base.memsearch.bytesequence; +import ghidra.util.bytesearch.ByteSequence; + public class ByteArrayByteSequence implements ByteSequence { private final byte[] bytes; diff --git a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequenceTest.java b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequenceTest.java index b4385acb52..adfee49544 100644 --- a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequenceTest.java +++ b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequenceTest.java @@ -19,6 +19,9 @@ import static org.junit.Assert.*; import org.junit.Test; +import ghidra.util.bytesearch.ByteSequence; +import ghidra.util.bytesearch.ExtendedByteSequence; + public class ByteArrayByteSequenceTest { private ByteSequence main = new ByteArrayByteSequence((byte) 0, (byte) 1, (byte) 2, (byte) 3); private ByteSequence extra = new ByteArrayByteSequence((byte) 4, (byte) 5); diff --git a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/MaskedBytesSequenceByteMatcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/MaskedBytesSequenceByteMatcherTest.java index ff9e19deee..1cb14e6c0e 100644 --- a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/MaskedBytesSequenceByteMatcherTest.java +++ b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/MaskedBytesSequenceByteMatcherTest.java @@ -23,7 +23,7 @@ import org.junit.Before; import org.junit.Test; import ghidra.features.base.memsearch.matcher.*; -import ghidra.util.bytesearch.Match; +import ghidra.util.bytesearch.*; public class MaskedBytesSequenceByteMatcherTest { diff --git a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/RegExByteMatcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/RegExByteMatcherTest.java index 1c91a714a1..b76ffb86b4 100644 --- a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/RegExByteMatcherTest.java +++ b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/RegExByteMatcherTest.java @@ -23,7 +23,7 @@ import org.junit.Before; import org.junit.Test; import ghidra.features.base.memsearch.matcher.*; -import ghidra.util.bytesearch.Match; +import ghidra.util.bytesearch.*; public class RegExByteMatcherTest { private ExtendedByteSequence byteSequence; diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java new file mode 100644 index 0000000000..b31f7ab7b0 --- /dev/null +++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java @@ -0,0 +1,273 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +import static org.junit.Assert.*; + +import java.io.*; +import java.util.*; + +import org.junit.Before; +import org.junit.Test; + +import ghidra.util.task.TaskMonitor; + +public class BulkPatternSearcherTest { + String data = "abbcabaaabbbcccba"; + + private TestPattern a = new TestPattern("a"); + private TestPattern ab = new TestPattern("ab"); + private TestPattern abc = new TestPattern("abc"); + private TestPattern cab = new TestPattern("cab"); + private TestPattern bcc = new TestPattern("bcc"); + + private BulkPatternSearcher searcher; + + @Before + public void setUp() { + List patterns = List.of(a, ab, abc, cab, bcc); + searcher = new BulkPatternSearcher(patterns); + } + + @Test + public void testMatchWithIntoList() { + List> results = new ArrayList>(); + searcher.search(data.getBytes(), results); + Iterator> it = results.iterator(); + assertTrue(it.hasNext()); + assertMatch(it.next(), a, 0); + assertMatch(it.next(), ab, 0); + assertMatch(it.next(), cab, 3); + assertMatch(it.next(), a, 4); + assertMatch(it.next(), ab, 4); + assertMatch(it.next(), a, 6); + assertMatch(it.next(), a, 7); + assertMatch(it.next(), a, 8); + assertMatch(it.next(), ab, 8); + assertMatch(it.next(), bcc, 11); + assertMatch(it.next(), a, 16); + assertFalse(it.hasNext()); + } + + @Test + public void testMatchWithIntoListWithBufferLimit() { + List> results = new ArrayList>(); + searcher.search(data.getBytes(), 5, results); + Iterator> it = results.iterator(); + assertTrue(it.hasNext()); + assertMatch(it.next(), a, 0); + assertMatch(it.next(), ab, 0); + assertMatch(it.next(), a, 4); + assertFalse(it.hasNext()); + } + + @Test + public void testMatchWithIterator() { + Iterator> it = searcher.search(data.getBytes()); + assertTrue(it.hasNext()); + assertMatch(it.next(), a, 0); + assertMatch(it.next(), ab, 0); + assertMatch(it.next(), cab, 3); + assertMatch(it.next(), a, 4); + assertMatch(it.next(), ab, 4); + assertMatch(it.next(), a, 6); + assertMatch(it.next(), a, 7); + assertMatch(it.next(), a, 8); + assertMatch(it.next(), ab, 8); + assertMatch(it.next(), bcc, 11); + assertMatch(it.next(), a, 16); + assertFalse(it.hasNext()); + + } + + @Test + public void testMatchWithIteratorAndBufferLimit() { + Iterator> it = searcher.search(data.getBytes(), 5); + assertTrue(it.hasNext()); + assertMatch(it.next(), a, 0); + assertMatch(it.next(), ab, 0); + assertMatch(it.next(), a, 4); + assertFalse(it.hasNext()); + } + + @Test + public void testInputStream() throws IOException { + TestPattern t = new TestPattern("test"); + TestPattern i = new TestPattern("input stream"); + TestPattern s = new TestPattern("stream"); + BulkPatternSearcher Matcher = new BulkPatternSearcher<>(List.of(t, i, s)); + + String input = "This is a test of the input stream"; + InputStream is = new ByteArrayInputStream(input.getBytes()); + + List> results = new ArrayList<>(); + Matcher.search(is, results, TaskMonitor.DUMMY); + + assertEquals(3, results.size()); + assertMatch(results.get(0), t, 10); + assertMatch(results.get(1), i, 22); + assertMatch(results.get(2), s, 28); + } + + @Test + public void testInputStreamWithMatchThatSpansBuffer() throws IOException { + TestPattern p1 = new TestPattern("test"); + TestPattern p2 = new TestPattern("test of the"); + TestPattern p3 = new TestPattern("stream"); + BulkPatternSearcher matcher = new BulkPatternSearcher<>(List.of(p1, p2, p3)); + matcher.setBufferSize(15); // test with buffer so that a pattern crosses buffer boundary + String input = "This is a test of the input stream"; + InputStream is = new ByteArrayInputStream(input.getBytes()); + + List> results = new ArrayList<>(); + matcher.search(is, -1, results, TaskMonitor.DUMMY); + + assertEquals(3, results.size()); + assertMatch(results.get(0), p1, 10); + assertMatch(results.get(1), p2, 10); + assertMatch(results.get(2), p3, 28); + + } + + @Test + public void testInputStreamWithMaxReadSet() throws IOException { + TestPattern t = new TestPattern("test"); + TestPattern i = new TestPattern("input stream"); + TestPattern s = new TestPattern("stream"); + BulkPatternSearcher matcher = new BulkPatternSearcher<>(List.of(t, i, s)); + + String input = "This is a test of the input stream"; + InputStream is = new ByteArrayInputStream(input.getBytes()); + + List> results = new ArrayList<>(); + matcher.search(is, 24, results, TaskMonitor.DUMMY); + + assertEquals(2, results.size()); + assertMatch(results.get(0), t, 10); + assertMatch(results.get(1), i, 22); + } + + @Test + public void testDittedPattern() { + TestPattern p1 = new TestPattern("b.t"); + TestPattern p2 = new TestPattern("t..t"); + TestPattern p3 = new TestPattern(".ba."); + searcher = new BulkPatternSearcher<>(List.of(p1, p2, p3)); + + String input = "bat baat bt abbt"; + Iterator> it = searcher.search(input.getBytes()); + assertTrue(it.hasNext()); + assertMatch(it.next(), p1, 0); + assertMatch(it.next(), p3, 3); + assertMatch(it.next(), p2, 7); + assertMatch(it.next(), p1, 13); + assertFalse(it.hasNext()); + } + + @Test + public void testStatesFullyDedup() { + TestPattern p1 = new TestPattern("..ab"); + TestPattern p2 = new TestPattern("..ac"); + TestPattern p3 = new TestPattern("axad"); + searcher = new BulkPatternSearcher<>(List.of(p1, p2, p3)); + + /** + This should produce the following search state graph. Each state indicates the + active patterns and the level (the # input bytes matched) and is shown as: + + (pattern, ..., level) + + (p1,p2,p3,0) + / \ + a !a (any input other than a) + / \ + / \ + (p1,p2,p3,1) (p1,p2,1) + / \ \ + x !x (any input) + / \ \ + / \ \ + / \ \ + / \ \ + (p1,p2,p3,2) (p1,p2,2) (p1,p2,2) dup + / \ / \ + a !a a !a + / \ / \ + (p1,p2,p3,3) null (p1,p2,3) null + / | \ / \ + b c d b c + / | \ / \ + (p1,4) (p2,4) (p3,4) (p1,4) dup (p2,4) dup + */ + + assertEquals(10, searcher.getUniqueStateCount()); + + } + + @Test + public void testSearchBeginningOnly() { + List> results = new ArrayList>(); + searcher.matches(data.getBytes(), data.length(), results); + Iterator> it = results.iterator(); + assertTrue(it.hasNext()); + assertMatch(it.next(), a, 0); + assertMatch(it.next(), ab, 0); + assertFalse(it.hasNext()); + } + + private void assertMatch(Match match, TestPattern expectedPattern, int start) { + assertEquals(new Match<>(expectedPattern, start, expectedPattern.getSize()), match); + } + + private class TestPattern extends DittedBitSequence { + + private String inputString; + + public TestPattern(String inputString) { + super(getBytes(inputString), getMask(inputString)); + this.inputString = inputString; + } + + private static byte[] getMask(String inputString) { + byte[] mask = new byte[inputString.length()]; + for (int i = 0; i < inputString.length(); i++) { + if (inputString.charAt(i) == '.') { + mask[i] = 0; + } + else { + mask[i] = (byte) 0xff; + } + } + return mask; + } + + private static byte[] getBytes(String inputString) { + byte[] bytes = inputString.getBytes(); + for (int i = 0; i < inputString.length(); i++) { + if (inputString.charAt(i) == '.') { + bytes[i] = 0; + } + } + return bytes; + } + + @Override + public String toString() { + return inputString; + } + + } +} diff --git a/Ghidra/Features/BytePatterns/ghidra_scripts/DumpMissedStarts.java b/Ghidra/Features/BytePatterns/ghidra_scripts/DumpMissedStarts.java index bc5278a474..39730e4508 100644 --- a/Ghidra/Features/BytePatterns/ghidra_scripts/DumpMissedStarts.java +++ b/Ghidra/Features/BytePatterns/ghidra_scripts/DumpMissedStarts.java @@ -30,31 +30,35 @@ import ghidra.util.constraint.ProgramDecisionTree; public class DumpMissedStarts extends GhidraScript implements PatternFactory { private static int bufsize = 20; private DummyMatchAction dummyaction; - private SequenceSearchState root; + private BulkPatternSearcher patternSearcher; private Memory memory; private byte[] bytebuffer; - ArrayList matchlist; + ArrayList> matchlist; - private boolean functionMatchesPattern(byte[] buff, int numbytes) { + private boolean functionMatchesPattern(byte[] buff, int numBytes) { matchlist.clear(); - root.sequenceMatch(buff, numbytes, matchlist); - if (matchlist.size() > 0) + patternSearcher.matches(buff, numBytes, matchlist); + if (matchlist.size() > 0) { return true; + } return false; } private boolean detectThunk(Function func, CodeUnit cu) { - if (cu == null) + if (cu == null) { return true; - if (cu instanceof Data) + } + if (cu instanceof Data) { return true; + } return false; } private void writeBytes(Writer w, byte[] buffer, int numbytes) throws IOException { StringBuffer buf = new StringBuffer(); - for (int i = 0; i < numbytes; ++i) + for (int i = 0; i < numbytes; ++i) { buf.append(Integer.toHexString(buffer[i] & 0xff)).append(' '); + } buf.append('\n'); w.write(buf.toString()); } @@ -71,18 +75,20 @@ public class DumpMissedStarts extends GhidraScript implements PatternFactory { ProgramDecisionTree patternDecisionTree = Patterns.getPatternDecisionTree(); ResourceFile[] fileList = Patterns.findPatternFiles(currentProgram, patternDecisionTree); ArrayList patternlist = new ArrayList<>(); - for (int i = 0; i < fileList.length; ++i) + for (int i = 0; i < fileList.length; ++i) { Pattern.readPostPatterns(fileList[i].getFile(true), patternlist, this); + } FileWriter fileWriter = new FileWriter(file); - root = SequenceSearchState.buildStateMachine(patternlist); + patternSearcher = new BulkPatternSearcher<>(patternlist); FunctionManager functionManager = currentProgram.getFunctionManager(); FunctionIterator iter = functionManager.getFunctions(true); while (iter.hasNext()) { Function func = iter.next(); CodeUnit cu = listing.getCodeUnitAt(func.getEntryPoint()); - if (detectThunk(func, cu)) + if (detectThunk(func, cu)) { continue; + } int numbytes = memory.getBytes(func.getEntryPoint(), bytebuffer); if ((numbytes > 0) && (!functionMatchesPattern(bytebuffer, numbytes))) { writeBytes(fileWriter, bytebuffer, numbytes); @@ -98,8 +104,9 @@ public class DumpMissedStarts extends GhidraScript implements PatternFactory { @Override public PostRule getPostRuleByName(String nm) { - if (nm.equals("align")) + if (nm.equals("align")) { return new AlignRule(); + } return null; } diff --git a/Ghidra/Features/BytePatterns/ghidra_scripts/PatternStats.java b/Ghidra/Features/BytePatterns/ghidra_scripts/PatternStats.java index 61f15c4846..1c3c001715 100644 --- a/Ghidra/Features/BytePatterns/ghidra_scripts/PatternStats.java +++ b/Ghidra/Features/BytePatterns/ghidra_scripts/PatternStats.java @@ -52,7 +52,7 @@ public class PatternStats extends GhidraScript implements PatternFactory { private MatchActionMarker codeBoundary = new MatchActionMarker(MatchActionMarker.CODE_BOUNDARY); private MatchActionMarker context = new MatchActionMarker(MatchActionMarker.CONTEXT); - private SequenceSearchState root; + private BulkPatternSearcher patternSearcher; private ArrayList accumList; private FunctionManager functionManager; private Listing listing; @@ -76,6 +76,7 @@ public class PatternStats extends GhidraScript implements PatternFactory { @Override public void apply(Program program, Address addr, Match match) { + // do nothing } @Override @@ -255,7 +256,7 @@ public class PatternStats extends GhidraScript implements PatternFactory { if (patternlist.size() == 0) { return; } - root = SequenceSearchState.buildStateMachine(patternlist); + patternSearcher = new BulkPatternSearcher<>(patternlist); accumList = new ArrayList<>(); for (int i = 0; i < patternlist.size(); ++i) { accumList.add(new PatternAccumulate(patternlist.get(i))); @@ -329,7 +330,7 @@ public class PatternStats extends GhidraScript implements PatternFactory { taskMonitor.setProgress(0); ArrayList> mymatches = new ArrayList<>(); long streamoffset = block.getStart().getOffset(); - root.apply(block.getData(), mymatches, taskMonitor); + patternSearcher.search(block.getData(), mymatches, taskMonitor); if (taskMonitor.isCancelled()) { return; } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java index 83eec4921c..82d0d1beaf 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java @@ -60,9 +60,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa private final static boolean OPTION_DEFAULT_BOOKMARKS = false; private static ProgramDecisionTree patternDecisitionTree; - // always need to initialize the root. - SequenceSearchState rootState = null; - SequenceSearchState explicitState = null; //for use during dynamic function start pattern discovery + // always need to initialize the pattern searcher. + BulkPatternSearcher patternSearcher = null; + BulkPatternSearcher explicitSearcher = null; //for use during dynamic function start pattern discovery private boolean executableBlocksOnly = true; // true if we only analyze executable blocks @@ -91,7 +91,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa } return patternDecisitionTree; } - + public ProgramDecisionTree getPatternDecisionTree() { return initializePatternDecisionTree(); } @@ -104,39 +104,32 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa this(name, DESCRIPTION, analyzerType); } - + public FunctionStartAnalyzer(String name, String description, AnalyzerType analyzerType) { super(name, description, analyzerType); - + setPriority(AnalysisPriority.CODE_ANALYSIS.after().after()); setDefaultEnablement(true); setSupportsOneTimeAnalysis(); } /** - * Sets the {@link SequenceSearchState}. Use this method when you've created a - * {@link SequenceSearchState} that you want to apply to the program. If you don't set - * the state explicitly, Ghidra will create one from the appropriate pattern file in - * {@link SequenceSearchState#initialize} - * @param explicit + * Sets the {@link BulkPatternSearcher}. Use this method when you've created a + * {@link BulkPatternSearcher} that you want to apply to the program. If you don't set + * the state explicitly, Ghidra will create one from the appropriate pattern file. + * @param searcher the explicit BulkPatternSearcher to use */ - public void setExplicitState(SequenceSearchState explicit) { - explicitState = explicit; + public void setExplicitState(BulkPatternSearcher searcher) { + explicitSearcher = searcher; } /** - * Clears the explict state. + * Clears the explicit state. */ public void clearExplicitState() { - explicitState = null; + explicitSearcher = null; } - /** - * apply any latent context at the location - * - * @param program - * @param addr - */ private void setCurrentContext(Program program, Address addr) { if (contextValueList == null) { return; @@ -159,7 +152,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa contextValueList = null; } - private void setDisassemblerContext(Program program, PseudoDisassemblerContext pcont, Address addr) { + private void setDisassemblerContext(Program program, PseudoDisassemblerContext pcont, + Address addr) { if (contextValueList == null) { return; } @@ -203,7 +197,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa private static final int MUST_HAVE_VALID_INSTRUCTIONS_NO_MIN = -1; // no minimum private static final int VALID_INSTRUCTIONS_NO_MAX = -1; // no maximum on instructions to check private static final int NO_VALID_INSTRUCTIONS_REQUIRED = 0; - + private String afterName = null; private int validCodeMin = NO_VALID_INSTRUCTIONS_REQUIRED; private int validCodeMax = VALID_INSTRUCTIONS_NO_MAX; @@ -233,12 +227,12 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa if (block == null) { return false; } - Matcher m = sectionNamePattern.matcher(block.getName()); - if (!m.matches()) { + Matcher m = sectionNamePattern.matcher(block.getName()); + if (!m.matches()) { return false; } } - + /** * If the match's mark point occurs in undefined data, schedule disassembly * and a function start at that address. If the match's mark point occurs at an instruction, but that @@ -264,21 +258,23 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa PseudoDisassembler pseudoDisassembler = new PseudoDisassembler(program); PseudoDisassemblerContext pcont = new PseudoDisassemblerContext(program.getProgramContext()); - + setDisassemblerContext(program, pcont, addr); boolean isvalid = false; if (validCodeMin == -1) { if (validCodeMax > 0) { // check at most N instructions pseudoDisassembler.setMaxInstructions(validCodeMax); } - isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, true, contiguous); + isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, true, + contiguous); } else { if (validCodeMax > 0) { // check at most N instructions pseudoDisassembler.setMaxInstructions(validCodeMax); } // disassemble only fallthru, must have validcode number of instructions - isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, false, contiguous); + isvalid = pseudoDisassembler.checkValidSubroutine(addr, pcont, true, false, + contiguous); int instrCount = pseudoDisassembler.getLastCheckValidInstructionCount(); if (instrCount < validCodeMin) { isvalid = false; @@ -443,7 +439,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa // if there are only pure data references to the location return pureDataReferencesOnly(program, addr); } - + } return true; } @@ -456,7 +452,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa * @return true if there are only pure data references (no flow, or r/w) */ private boolean pureDataReferencesOnly(Program program, Address addrToCheck) { - ReferenceIterator referencesTo = program.getReferenceManager().getReferencesTo(addrToCheck); + ReferenceIterator referencesTo = + program.getReferenceManager().getReferencesTo(addrToCheck); if (!referencesTo.hasNext()) { return false; } @@ -486,12 +483,13 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa Function funcAbove = getFunctionAbove(program, addr); return checkAlreadyInFunctionAbove(program, addr, funcAbove); } - + /* * Check if in a function above * return true if already in function above, false otherwise even if in another function */ - private boolean checkAlreadyInFunctionAbove(Program program, Address addr, Function funcAbove) { + private boolean checkAlreadyInFunctionAbove(Program program, Address addr, + Function funcAbove) { // if no funcAbove, make sure an instruction, doesn't fall into this one. Address addrBefore = addr.previous(); if (addrBefore == null) { @@ -530,13 +528,13 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa return false; } - + /** * Get an existing function right above the addr. * @param program program to check * @param addr address to check * @return true if there is an existing function above addr - */ + */ private Function getFunctionAbove(Program program, Address addr) { // make sure there is an end of function before this one, and addr is not in the function Function func = null; @@ -570,7 +568,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa String attrValue = attributes.get(attrName); attrName = attrName.toLowerCase(); switch (attrName) { - case "after": + case "after": afterName = attrValue; if (afterName.startsWith("func")) { hasCodeConstraints = true; @@ -592,8 +590,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa "funcstart pattern attribute 'after' must be one of 'function', 'instruction', 'data', 'defined'"); } break; - - // set check for valid code and the minimum number of instructions required + + // set check for valid code and the minimum number of instructions required // if no maximum is set, then the instructions MUST be fallthru instructions, don't check branch flows case "validcode": String validcodeStr = attrValue; @@ -617,9 +615,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa validCodeMax = validCodeMin; } break; - - // set the maximum number of instructions to check - // if maximum is set, then allow non fallthru instructions while flowing + + // set the maximum number of instructions to check + // if maximum is set, then allow non fallthru instructions while flowing case "validcodemax": String validcodeMaxStr = attrValue; // check up instructions for valid code @@ -630,39 +628,41 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa validCodeMin = MUST_HAVE_VALID_INSTRUCTIONS_NO_MIN; } break; - + // minimum number of instructions for validcode must be contiguous instructions case "contiguous": - String fallThruOnlyStr = attrValue; - // check up instructions for valid code + String fallThruOnlyStr = attrValue; + // check up instructions for valid code + contiguous = true; + if (fallThruOnlyStr.equalsIgnoreCase("false")) { + contiguous = false; + } + else if (fallThruOnlyStr.equalsIgnoreCase("true")) { contiguous = true; - if (fallThruOnlyStr.equalsIgnoreCase("false")) { - contiguous = false; - } - else if (fallThruOnlyStr.equalsIgnoreCase("true")) { - contiguous = true; - } else { - Msg.error(this, "Bad contiguous option (true,false): " + attrName + " = " + attrValue); - } - break; - + } + else { + Msg.error(this, "Bad contiguous option (true,false): " + attrName + + " = " + attrValue); + } + break; + case "label": String name = attrValue; label = name; break; - + case "thunk": isThunk = true; break; - + case "section": sectionNamePattern = java.util.regex.Pattern.compile(attrValue); break; - + case "noreturn": noreturn = true; break; - + // TODO: add the ability to make data based on a pattern of bytes // useful after defined instructions/functions to take up filler byte patterns // will allow more finding of code that is after defined data @@ -671,7 +671,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa // // create undefined data of the given size // makeData = Integer.parseInt(validcodeDataStr); // break; - + default: Msg.error(this, "Unknown Patten option: " + attrName + " = " + attrValue); } @@ -760,7 +760,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa } } - + @Override public boolean canAnalyze(Program program) { ProgramDecisionTree patternDecisionTree = getPatternDecisionTree(); @@ -795,8 +795,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log) throws CancelledException { - SequenceSearchState root = initialize(program); - if (root == null) { + BulkPatternSearcher searcher = initialize(program); + if (searcher == null) { String message = "Could not initialize a search state."; log.appendMsg(getName(), message); log.setStatus(message); @@ -810,27 +810,27 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa // this will keep cruft from accumulating in the property map. getOrCreatePotentialMatchPropertyMap(program).remove(set); - MemoryBytePatternSearcher patternSearcher; - patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) { + MemoryBytePatternSearcher memorySearcher = + new MemoryBytePatternSearcher("Function Starts", searcher) { - @Override - public void preMatchApply(MatchAction[] actions, Address addr) { - contextValueList = null; // make sure, only context from these actions used - } - - @Override - public void postMatchApply(MatchAction[] actions, Address addr) { - // Actions might have set context, check if postcondition failed first - if (!postreqFailedResult.contains(addr)) { - setCurrentContext(program, addr); + @Override + public void preMatchApply(MatchAction[] actions, Address addr) { + contextValueList = null; // make sure, only context from these actions used } - // get rid of the context list. - contextValueList = null; - } - }; - patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly); - patternSearcher.search(program, set, monitor); + @Override + public void postMatchApply(MatchAction[] actions, Address addr) { + // Actions might have set context, check if postcondition failed first + if (!postreqFailedResult.contains(addr)) { + setCurrentContext(program, addr); + } + // get rid of the context list. + contextValueList = null; + } + }; + memorySearcher.setSearchExecutableOnly(doExecutableBlocksOnly); + + memorySearcher.search(program, set, monitor); AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program); if (!disassemResult.isEmpty()) { @@ -838,7 +838,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa AddressSet doNowDisassembly = disassemResult.intersect(funcResult); // this will disassemble at this analyzers priority analysisManager.disassemble(doNowDisassembly); - + // delay disassemble of possible function starts AddressSet delayedDisassembly = disassemResult.subtract(funcResult); analysisManager.disassemble(delayedDisassembly, AnalysisPriority.DISASSEMBLY); @@ -894,7 +894,7 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa setbookmark = options.getBoolean(OPTION_NAME_BOOKMARKS, setbookmark); } - protected SequenceSearchState initialize(Program program) { + protected BulkPatternSearcher initialize(Program program) { potentialFuncResult = new AddressSet(); disassemResult = new AddressSet(); @@ -902,15 +902,15 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa postreqFailedResult = new AddressSet(); funcResult = new AddressSet(); - if (explicitState != null) { - return explicitState; + if (explicitSearcher != null) { + return explicitSearcher; } // TODO: Check the times on the patterns files, maybe reload them! // could get times of all files and record them to check times. // filelist keeps getting re-parsed...! - if (rootState != null) { - return rootState; + if (patternSearcher != null) { + return patternSearcher; } ArrayList patternlist = new ArrayList<>(); @@ -930,9 +930,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa return null; } - SequenceSearchState root = SequenceSearchState.buildStateMachine(patternlist); - - return root; + BulkPatternSearcher searcher = new BulkPatternSearcher<>(patternlist); + return searcher; } private ArrayList readPatterns(ResourceFile[] filelist, Program program) { @@ -987,7 +986,8 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa final class PossibleDelayedFunctionCreator extends AnalyzerAdapter { PossibleDelayedFunctionCreator() { - super(FunctionStartAnalyzer.FUNCTION_START_SEARCH + " delayed", AnalysisPriority.DATA_ANALYSIS.after()); + super(FunctionStartAnalyzer.FUNCTION_START_SEARCH + " delayed", + AnalysisPriority.DATA_ANALYSIS.after()); } @Override @@ -1001,7 +1001,7 @@ final class PossibleDelayedFunctionCreator extends AnalyzerAdapter { if (hasConditionalReferences(addedProgram, address)) { continue; } - + // Check for any function containing the potential start detected earlier in analysis Function funcAt = addedProgram.getFunctionManager().getFunctionContaining(address); @@ -1017,7 +1017,7 @@ final class PossibleDelayedFunctionCreator extends AnalyzerAdapter { } functionStarts.add(address); } - + // create functions that still don't exist/overlap new CreateFunctionCmd(functionStarts, false).applyTo(addedProgram, addedMonitor); return true; diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartDataPostAnalyzer.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartDataPostAnalyzer.java index 4359ceb300..215db7a78e 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartDataPostAnalyzer.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartDataPostAnalyzer.java @@ -1,13 +1,12 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +18,8 @@ package ghidra.app.analyzers; import ghidra.app.services.AnalysisPriority; import ghidra.app.services.AnalyzerType; import ghidra.program.model.listing.Program; -import ghidra.util.bytesearch.SequenceSearchState; +import ghidra.util.bytesearch.BulkPatternSearcher; +import ghidra.util.bytesearch.Pattern; public class FunctionStartDataPostAnalyzer extends FunctionStartAnalyzer { protected static final String FUNCTION_START_POST_SEARCH = "Function Start Post Search"; @@ -35,13 +35,13 @@ public class FunctionStartDataPostAnalyzer extends FunctionStartAnalyzer { if (!super.canAnalyze(program)) { return false; } - SequenceSearchState localRoot = initialize(program); - if (localRoot == null) { + BulkPatternSearcher localSearcher = initialize(program); + if (localSearcher == null) { return false; } if (hasDataConstraints) { - // cache the localRoot - rootState = localRoot; + // cache the pattern searcher + patternSearcher = localSearcher; return true; } return false; diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartFuncAnalyzer.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartFuncAnalyzer.java index 9eff360632..d4a1bae392 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartFuncAnalyzer.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartFuncAnalyzer.java @@ -21,7 +21,8 @@ import ghidra.app.util.importer.MessageLog; import ghidra.program.model.address.AddressSet; import ghidra.program.model.address.AddressSetView; import ghidra.program.model.listing.Program; -import ghidra.util.bytesearch.SequenceSearchState; +import ghidra.util.bytesearch.BulkPatternSearcher; +import ghidra.util.bytesearch.Pattern; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; @@ -48,7 +49,7 @@ public class FunctionStartFuncAnalyzer extends FunctionStartAnalyzer { if (set.isEmpty()) { return true; } - + return super.added(program, set, monitor, log); } @@ -57,13 +58,13 @@ public class FunctionStartFuncAnalyzer extends FunctionStartAnalyzer { if (!super.canAnalyze(program)) { return false; } - SequenceSearchState localRoot = initialize(program); - if (localRoot == null) { + BulkPatternSearcher localSearcher = initialize(program); + if (localSearcher == null) { return false; } if (hasFunctionStartConstraints) { - // cache the localRoot - rootState = localRoot; + // cache the local pattern searcher + patternSearcher = localSearcher; return true; } return false; diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartPostAnalyzer.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartPostAnalyzer.java index 8730606c3a..f35dc97e9c 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartPostAnalyzer.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartPostAnalyzer.java @@ -1,13 +1,12 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +18,8 @@ package ghidra.app.analyzers; import ghidra.app.services.AnalysisPriority; import ghidra.app.services.AnalyzerType; import ghidra.program.model.listing.Program; -import ghidra.util.bytesearch.SequenceSearchState; +import ghidra.util.bytesearch.BulkPatternSearcher; +import ghidra.util.bytesearch.Pattern; public class FunctionStartPostAnalyzer extends FunctionStartAnalyzer { protected static final String FUNCTION_START_POST_SEARCH = "Function Start Post Search"; @@ -35,13 +35,13 @@ public class FunctionStartPostAnalyzer extends FunctionStartAnalyzer { if (!super.canAnalyze(program)) { return false; } - SequenceSearchState localRoot = initialize(program); - if (localRoot == null) { + BulkPatternSearcher localSearcher = initialize(program); + if (localSearcher == null) { return false; } if (hasCodeConstraints || hasDataConstraints) { - // cache the localRoot - rootState = localRoot; + // cache the local pattern searcher + patternSearcher = localSearcher; return true; } return false; diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/bitpatterns/gui/ClipboardPanel.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/bitpatterns/gui/ClipboardPanel.java index fa96fc2466..e5c7fce9f5 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/bitpatterns/gui/ClipboardPanel.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/bitpatterns/gui/ClipboardPanel.java @@ -65,10 +65,6 @@ public class ClipboardPanel extends JPanel { private boolean onlyPrePatterns; - /** - * Class for building the pattern clipboard - * @param plugin - */ public ClipboardPanel(FunctionBitPatternsExplorerPlugin plugin) { super(); BoxLayout mainLayout = new BoxLayout(this, BoxLayout.Y_AXIS); @@ -121,8 +117,8 @@ public class ClipboardPanel extends JPanel { MatchAction[] actions = getMatchActions(funcStartAnalyzer, pattern); pattern.setMatchActions(actions); } - SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList); - funcStartAnalyzer.setExplicitState(root); + BulkPatternSearcher searcher = new BulkPatternSearcher<>(patternList); + funcStartAnalyzer.setExplicitState(searcher); AutoAnalysisManager autoManager = AutoAnalysisManager.getAnalysisManager(currentProgram); autoManager.scheduleOneTimeAnalysis(funcStartAnalyzer, @@ -214,7 +210,7 @@ public class ClipboardPanel extends JPanel { Msg.showWarn(this, this, "Only Pre-Patterns", "Only Pre-Patterns in selection: no true/false positive information will be calculated."); } - SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList); + BulkPatternSearcher searcher = new BulkPatternSearcher<>(patternList); indexToSize.clear(); for (Pattern pattern : patternList) { indexToSize.put(pattern.getIndex(), pattern.getSize()); @@ -230,17 +226,17 @@ public class ClipboardPanel extends JPanel { if (!block.isExecute()) { continue; } - searchBlock(root, block, matchStats, currentProgram, TaskMonitor.DUMMY); + searchBlock(searcher, block, matchStats, currentProgram, TaskMonitor.DUMMY); } return matchStats; } - private void searchBlock(SequenceSearchState root, MemoryBlock block, + private void searchBlock(BulkPatternSearcher searcher, MemoryBlock block, PatternEvaluationStats matchStats, Program program, TaskMonitor monitor) { ArrayList> mymatches = new ArrayList<>(); try { - root.apply(block.getData(), mymatches, monitor); + searcher.search(block.getData(), mymatches, monitor); } catch (IOException e) { e.printStackTrace(); @@ -309,6 +305,7 @@ public class ClipboardPanel extends JPanel { addSeparator(pattern.getHexString(), index), funcStart, postBits, totalBits); matchStats.addRowObject(rowObject); } + private int getNumPostBits(Pattern pattern) { int marked = pattern.getMarkOffset(); if (marked == 0) { @@ -316,6 +313,7 @@ public class ClipboardPanel extends JPanel { } return pattern.getNumFixedBits() - pattern.getNumInitialFixedBits(marked); } + private PatternMatchType getMatchType(Program program, Address funcStart, ContextRegisterFilter cRegFilter) { if (cRegFilter != null) { diff --git a/Ghidra/Processors/AARCH64/src/main/java/ghidra/app/plugin/core/analysis/AARCH64PltThunkAnalyzer.java b/Ghidra/Processors/AARCH64/src/main/java/ghidra/app/plugin/core/analysis/AARCH64PltThunkAnalyzer.java index b69e128b1c..8e0ad9a741 100644 --- a/Ghidra/Processors/AARCH64/src/main/java/ghidra/app/plugin/core/analysis/AARCH64PltThunkAnalyzer.java +++ b/Ghidra/Processors/AARCH64/src/main/java/ghidra/app/plugin/core/analysis/AARCH64PltThunkAnalyzer.java @@ -42,31 +42,31 @@ import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; public class AARCH64PltThunkAnalyzer extends AbstractAnalyzer { - + private static final String NAME = "AARCH64 ELF PLT Thunks"; private static final String DESCRIPTION = "Create AARM64 ELF PLT thunk functions"; private static final String PROCESSOR_NAME = "AARCH64"; - + private static final String PLT_THUNK_PATTERN_FILE = "aarch64-pltThunks.xml"; - + private static boolean patternLoadFailed; private static ArrayList leThunkPatterns; private static int maxPatternLength; - + private Register x17Reg; - + public AARCH64PltThunkAnalyzer() { super(NAME, DESCRIPTION, AnalyzerType.BYTE_ANALYZER); // assumes ELF Loader disassembled PLT section setDefaultEnablement(true); setPriority(AnalysisPriority.FORMAT_ANALYSIS); } - + @Override public boolean canAnalyze(Program program) { Language language = program.getLanguage(); // TODO: what about 32/64 hybrid case? if (PROCESSOR_NAME.equals(language.getProcessor().toString()) && - patternsLoaded(language.isBigEndian())) { + patternsLoaded(language.isBigEndian())) { x17Reg = program.getRegister("x17"); return x17Reg != null; } @@ -80,13 +80,13 @@ public class AARCH64PltThunkAnalyzer extends AbstractAnalyzer { if (leThunkPatterns != null) { return true; } - + try { ResourceFile patternFile = Application.getModuleDataFile(PLT_THUNK_PATTERN_FILE); - + leThunkPatterns = new ArrayList<>(); Pattern.readPatterns(patternFile, leThunkPatterns, null); - + maxPatternLength = 0; for (Pattern pattern : leThunkPatterns) { int len = pattern.getSize(); @@ -97,71 +97,77 @@ public class AARCH64PltThunkAnalyzer extends AbstractAnalyzer { maxPatternLength = len; } } - - } catch (FileNotFoundException e) { - Msg.error(AARCH64PltThunkAnalyzer.class, "AARCH64 resource file not found: " + PLT_THUNK_PATTERN_FILE); - patternLoadFailed = true; - return false; - } catch (SAXException | IOException e) { - Msg.error(AARCH64PltThunkAnalyzer.class, "Failed to parse byte pattern file: " + PLT_THUNK_PATTERN_FILE, e); + + } + catch (FileNotFoundException e) { + Msg.error(AARCH64PltThunkAnalyzer.class, + "AARCH64 resource file not found: " + PLT_THUNK_PATTERN_FILE); patternLoadFailed = true; return false; } - + catch (SAXException | IOException e) { + Msg.error(AARCH64PltThunkAnalyzer.class, + "Failed to parse byte pattern file: " + PLT_THUNK_PATTERN_FILE, e); + patternLoadFailed = true; + return false; + } + return true; } @Override public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log) throws CancelledException { - + Memory memory = program.getMemory(); MemoryBlock block = memory.getBlock(".plt"); if (block == null) { return true; } - + set = set.intersectRange(block.getStart(), block.getEnd()); set = removeFunctionBodies(program, set, monitor); if (set.isEmpty()) { return true; } - SequenceSearchState sequenceSearchState = SequenceSearchState.buildStateMachine( - leThunkPatterns); - + BulkPatternSearcher searcher = new BulkPatternSearcher<>(leThunkPatterns); + monitor.setIndeterminate(true); monitor.setProgress(0); - + ArrayList> matches = new ArrayList<>(); - + try { for (AddressRange range : set.getAddressRanges()) { - - byte[] bytes = new byte[(int)range.getLength()]; + + byte[] bytes = new byte[(int) range.getLength()]; if (block.getBytes(range.getMinAddress(), bytes, 0, bytes.length) != bytes.length) { log.appendMsg("Expected initialized .plt section block"); return false; } - + matches.clear(); - sequenceSearchState.apply(bytes, matches); - + searcher.search(bytes, matches); + for (Match match : matches) { Pattern pattern = match.getPattern(); - Address addr = range.getMinAddress().add(match.getStart() + pattern.getMarkOffset()); + Address addr = + range.getMinAddress().add(match.getStart() + pattern.getMarkOffset()); analyzePltThunk(program, addr, match.getLength(), monitor); } - + } - } catch (MemoryAccessException | AddressOutOfBoundsException e) { + } + catch (MemoryAccessException | AddressOutOfBoundsException e) { log.appendMsg("Expected initialized .plt section block: " + e.getMessage()); } - + return true; } - private AddressSetView removeFunctionBodies(Program program, AddressSetView set, TaskMonitor monitor) throws CancelledException { + private AddressSetView removeFunctionBodies(Program program, AddressSetView set, + TaskMonitor monitor) throws CancelledException { if (set.isEmpty()) { return set; } @@ -173,92 +179,102 @@ public class AARCH64PltThunkAnalyzer extends AbstractAnalyzer { return set; } - private void analyzePltThunk(Program program, Address entryAddr, int thunkSize, TaskMonitor monitor) + private void analyzePltThunk(Program program, Address entryAddr, int thunkSize, + TaskMonitor monitor) throws CancelledException { - + SymbolicPropogator symEval = new SymbolicPropogator(program, false); symEval.setParamRefCheck(false); symEval.setReturnRefCheck(false); symEval.setStoredRefCheck(false); - + AddressSet thunkBody = new AddressSet(entryAddr, entryAddr.add(thunkSize - 1)); - + ContextEvaluator eval = new ContextEvaluatorAdapter() { @Override public boolean followFalseConditionalBranches() { return false; // should never happen - just in case } - + @Override - public boolean evaluateReference(VarnodeContext context, Instruction instr, int pcodeop, Address address, + public boolean evaluateReference(VarnodeContext context, Instruction instr, int pcodeop, + Address address, int size, DataType dataType, RefType refType) { return true; } - + @Override public boolean evaluateDestination(VarnodeContext context, Instruction instruction) { - + // We only handle indirect branch through x17 register - if (!"br".equals(instruction.getMnemonicString()) || !x17Reg.equals(instruction.getRegister(0))) { + if (!"br".equals(instruction.getMnemonicString()) || + !x17Reg.equals(instruction.getRegister(0))) { return true; } - + // Change br flow to call-return instruction.setFlowOverride(FlowOverride.CALL_RETURN); - + RegisterValue x17Value = context.getRegisterValue(x17Reg); if (x17Value != null && x17Value.hasValue()) { - Address destAddr = entryAddr.getNewAddress(x17Value.getUnsignedValue().longValue()); - Function thunkedFunction = createDestinationFunction(program, destAddr, instruction.getAddress(), + Address destAddr = + entryAddr.getNewAddress(x17Value.getUnsignedValue().longValue()); + Function thunkedFunction = + createDestinationFunction(program, destAddr, instruction.getAddress(), monitor); if (thunkedFunction != null) { - CreateThunkFunctionCmd cmd = new CreateThunkFunctionCmd(entryAddr, thunkBody, + CreateThunkFunctionCmd cmd = + new CreateThunkFunctionCmd(entryAddr, thunkBody, thunkedFunction.getEntryPoint()); cmd.applyTo(program); } } - + return true; } - + @Override public boolean allowAccess(VarnodeContext context, Address address) { return true; } }; - + symEval.flowConstants(entryAddr, thunkBody, eval, false, monitor); } - private Function createDestinationFunction(Program program, Address addr, Address flowFromAddr, TaskMonitor monitor) { + private Function createDestinationFunction(Program program, Address addr, Address flowFromAddr, + TaskMonitor monitor) { Listing listing = program.getListing(); BookmarkManager bookmarkMgr = program.getBookmarkManager(); - + if (!program.getMemory().contains(addr)) { - bookmarkMgr.setBookmark(flowFromAddr, BookmarkType.ERROR, "Bad Reference", "No memory for PLT Thunk destination at " + addr); + bookmarkMgr.setBookmark(flowFromAddr, BookmarkType.ERROR, "Bad Reference", + "No memory for PLT Thunk destination at " + addr); return null; } - + Function function = listing.getFunctionAt(addr); if (function != null) { return function; } - + CodeUnit cu = listing.getCodeUnitContaining(addr); if (cu == null) { throw new AssertException("expected code unit in memory"); } if (!addr.equals(cu.getMinAddress())) { - bookmarkMgr.setBookmark(cu.getMinAddress(), BookmarkType.ERROR, "Code Unit Conflict", - "Expected function entry at " + addr + " referenced by PLT Thunk at " + flowFromAddr); + bookmarkMgr.setBookmark(cu.getMinAddress(), BookmarkType.ERROR, "Code Unit Conflict", + "Expected function entry at " + addr + " referenced by PLT Thunk at " + + flowFromAddr); return null; } if (cu instanceof Data) { - Data d = (Data)cu; + Data d = (Data) cu; if (d.isDefined()) { - bookmarkMgr.setBookmark(addr, BookmarkType.ERROR, "Code Unit Conflict", "Expected function entry referenced by PLT Thunk at " + flowFromAddr); + bookmarkMgr.setBookmark(addr, BookmarkType.ERROR, "Code Unit Conflict", + "Expected function entry referenced by PLT Thunk at " + flowFromAddr); return null; } DisassembleCommand cmd = new DisassembleCommand(addr, null, true); @@ -266,7 +282,7 @@ public class AARCH64PltThunkAnalyzer extends AbstractAnalyzer { return null; } } - + CreateFunctionCmd cmd = new CreateFunctionCmd(addr); if (cmd.applyTo(program, monitor)) { return cmd.getFunction();