diff --git a/Ghidra/Features/Base/certification.manifest b/Ghidra/Features/Base/certification.manifest index 4bbc7a8048..d9a184e025 100644 --- a/Ghidra/Features/Base/certification.manifest +++ b/Ghidra/Features/Base/certification.manifest @@ -1096,3 +1096,6 @@ src/test.slow/resources/ghidra/app/script/GhidraScriptAsk.properties||GHIDRA|||| src/test/resources/defaultTools/TestCodeBrowser.tool||GHIDRA||||END| src/test/resources/ghidra/app/util/opinion/decompile_debug_test.xml||GHIDRA||||END| src/test/resources/ghidra/app/util/opinion/test.ord||GHIDRA||||END| +src/test/resources/ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml||GHIDRA||||END| +src/test/resources/ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml||GHIDRA||||END| +src/test/resources/ghidra/util/bytesearch/bytepatternsearch/parse_errors.xml||GHIDRA||||END| diff --git a/Ghidra/Features/Base/src/main/help/help/topics/Search/Searching.htm b/Ghidra/Features/Base/src/main/help/help/topics/Search/Searching.htm index a578da2b89..8c194a4f0e 100644 --- a/Ghidra/Features/Base/src/main/help/help/topics/Search/Searching.htm +++ b/Ghidra/Features/Base/src/main/help/help/topics/Search/Searching.htm @@ -39,7 +39,7 @@
  • Search For Direct References
    • -
  • Search For Instruction Patterns
    • +
  • Search For Instruction Patterns

    •  

    diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfig.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfig.java new file mode 100644 index 0000000000..b383847892 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfig.java @@ -0,0 +1,99 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +/** + * This class handles search configurations for Ghidra's Byte Pattern Searcher which dictate the + * types of data manipulations performed by {@link BpsTransformationEngine} required prior to search + * submission. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * Supported data manipulations are found in {@link BpsTransformationEngine}. + *

    + * Search configurations determine byte preparation manipulations prior to search submission. + */ +class BpsConfig { + + private boolean isSearchBE; + private boolean isSearchLE; + private boolean extendPattern; // Zero-extend patterns to fill larger word sizes + private int programWordSize; + + /** + * Default constructor initializing standard search settings; primarily used for search patterns + * which are 8 bits. Patterns of this size will not need splitting nor will they need to be + * manipulated to be LE as endianness of a pattern this size does not change the pattern. + * + * @param isByteExtension choice to zero-extend patterns to fit the word size of the program + * @param programWordSize word size of the program + */ + BpsConfig(int programWordSize, boolean isByteExtension) { + this.programWordSize = programWordSize; + this.extendPattern = isByteExtension; + } + + /** + * Constructor for handling search settings for patterns greater than 8 bits in size. + * + * @param programWordSize word size of the program + * @param isByteExtension choice to zero-extend patterns to fit the word size of the program + * @param isSearchBE choice to perform Big Endian search + * @param isSearchLE choice to perform Little Endian search + */ + public BpsConfig(int programWordSize, boolean isByteExtension, + boolean isSearchBE, boolean isSearchLE) { + this(programWordSize, isByteExtension); + this.isSearchBE = isSearchBE; + this.isSearchLE = isSearchLE; + } + + /** + * {@return big endian choice} + */ + boolean isSearchBE() { + return this.isSearchBE; + } + + /** + * {@return little endian choice} + */ + boolean isSearchLE() { + return this.isSearchLE; + } + + /** + * Search for patterns which have been zero-padded to fill the word size of the program. + *

    + * Note: Quantization of word sizes is performed - Example: if the byte pattern's + * word size is 16 (Word) but the program's word size is 64 (QWord), choosing to extend the data + * type will produce 2 additional byte arrays, zero-padded to 32 (DWord) and 64 (QWord). + * + * @return extend pattern choice + */ + boolean isExtendPattern() { + return this.extendPattern; + } + + /** + * {@return word size of the program} + */ + int getProgramWordSize() { + return this.programWordSize; + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfigBuilder.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfigBuilder.java new file mode 100644 index 0000000000..f2738d4432 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsConfigBuilder.java @@ -0,0 +1,100 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch.bytepatternsearch; + +/** + * A helper class to build up the potentially complicated {@link BpsConfig}. + *

    + * Manage the generation of a search configuration for use by Ghidra's Byte Pattern Search + * capability. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + */ +public class BpsConfigBuilder { + + private int programWordSize; + private boolean isByteExtension; + private boolean littleEndianSearch; + private boolean bigEndianSearch = true; + + /** + * Constructor. + * + * @param programWordSize the word size of the program; must be greater than 0 + */ + public BpsConfigBuilder(int programWordSize) { + + if (programWordSize <= 0) { + throw new IllegalArgumentException("Invalid program word size: " + programWordSize); + } + + this.programWordSize = programWordSize; + } + + /** + * Specify whether to perform a zero-extension of byte patterns to fit the program word size if + * the pattern's word size is smaller than the program's. + *

    + * A true setting will perform quantization of sizes to build out the pattern's word size + * iteratively until it fills the program's word size. See {@link BpsConfig} for details. + * + * @param isByteExtension true signals to zero-extend byte patterns to fill program word size + * + * @return this builder instance + */ + public BpsConfigBuilder setPerformExtension(boolean isByteExtension) { + this.isByteExtension = isByteExtension; + return this; + } + + /** + * Specify whether to search for patterns as Little Endian. + * + * @param isLeSearch true signals the transformation of patterns to be LE prior to + * search + * + * @return this builder instance + */ + public BpsConfigBuilder setLittleEndianSearch(boolean isLeSearch) { + this.littleEndianSearch = isLeSearch; + return this; + } + + /** + * Specify whether to search for patterns as Big Endian. Default is true. + * + * @param isBeSearch true signals patterns to be searched as BE (their assumed default + * endianness) + * + * @return this builder instance + */ + public BpsConfigBuilder setBigEndianSearch(boolean isBeSearch) { + this.bigEndianSearch = isBeSearch; + return this; + } + + /** + * Builds the final {@link BpsConfig}. + * @return the new search configuration + */ + public BpsConfig build() { + + // Default search configuration does not perform splitting + return new BpsConfig(this.programWordSize, this.isByteExtension, + this.bigEndianSearch, this.littleEndianSearch); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsPattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsPattern.java new file mode 100644 index 0000000000..fb1bd5e87f --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsPattern.java @@ -0,0 +1,140 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch.bytepatternsearch; + +import java.nio.ByteBuffer; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * This class represents a byte pattern in support of the {@link BpsXmlParser} of the Byte Pattern + * Search capability within Ghidra. + *

    + * The design of the XML Parser and its supporting classes can be found in {@link BpsXmlParser}. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This class represents a byte pattern object which is populated by {@link BpsXmlParser}. This + * object wraps a search byte array (inside a {@link ByteBuffer}), its parent {@link BpsSearchItem}, + * and its parsed XML tag name for use downstream. + * + *

    Object Variables

    + * + */ +public final class BpsPattern { + private final String name; + private ByteBuffer buffer; + private BpsSearchItem searchItem; + private final BpsTransformation transformation; + + /** + * Constructor. + * + * @param name parsed XML tag containing this byte pattern array + * @param rawBytes parsed byte array wrapped in a ByteBuffer + * @param parentSearchItem parent search item containing this pattern for searching + * @param transformationRecord byte manipulation order + */ + public BpsPattern(String name, ByteBuffer rawBytes, BpsSearchItem parentSearchItem, + BpsTransformation transformationRecord) { + this.name = name; + this.buffer = rawBytes.duplicate().rewind(); // Safeguard original data integrity + this.searchItem = parentSearchItem; + this.transformation = transformationRecord; + } + + /** + * {@return the name as parsed from the XML} + */ + public String getName() { + return name; + } + + /** + * {@return the parent search item containing this byte array} + */ + public BpsSearchItem getSearchItem() { + return searchItem; + } + + /** + * {@return the buffer containing the byte pattern. Downstream consumers are prevented from + * corrupting the pattern} + */ + public ByteBuffer getBuffer() { + return buffer.asReadOnlyBuffer().rewind(); + } + + /** + * {@return the transformation record of byte manipulations performed on the pattern} + */ + public BpsTransformation getTransformation() { + return transformation; + } + + @Override + public String toString() { + return getDescription(); + } + + /** + * {@return the description of the pattern} + */ + public String getDescription() { + ByteBuffer view = getBuffer(); + byte[] bytes = new byte[view.remaining()]; + view.get(bytes); + String hexBytes = IntStream.range(0, bytes.length) + .mapToObj(i -> String.format("%02X", bytes[i])) + .collect(Collectors.joining(" ")); + return String.format("[Pattern Name: %s | Word Size: %s] -> Pattern: %s | Form: [%s]", + searchItem.getPatternName(), + searchItem.getWordSize(), hexBytes, transformation.toString()); + + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearch.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearch.java new file mode 100644 index 0000000000..3a3ffe9fb3 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearch.java @@ -0,0 +1,150 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +import java.util.*; + +/** + * This class represents a collection of {@link BpsSearchItem}'s and their corresponding attributes + * in support of Ghidra's Byte Pattern Searcher capability. + *

    + * The design of the XML Parser and its supporting classes, as well as an explanation of the + * required XML format can be found in {@link BpsXmlParser}. + *

    + * The design of the Byte Pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This class manages a Search, or list, of {@link BpsSearchItem} objects parsed from the + * {@code } tag. A search is populated by {@link BpsXmlParser} and referenced for + * processing by downstream Byte Pattern Searcher components. + * + *

    Byte Pattern Search Configurations

    + *

    + * Required Variables: + *

    + * + * + *

    + * Optional Variables: + *

    + * Note: additional attributes may be included on the {@code} tag, they + * will be stored in a map for filtering and sorting downstream. + */ +public class BpsSearch { + private List searchItems; + private String searchName; + private String id; + private String description; + private String submitter; + private Map additionalAttributes; + + /** + * Constructor. + * + * @param searchName name identifying a Search + * @param generatedKey unique value across all BpsSearch and BpsSearchItem objects; used for + * skipping patterns at the {@code } and {@code } levels. + * @param searchItems belonging to this Search + */ + public BpsSearch(String searchName, String generatedKey, List searchItems) { + this.searchName = searchName; + this.id = generatedKey; + this.searchItems = searchItems; + this.additionalAttributes = new HashMap<>(); + } + + /** + * {@return the name of the Search} + */ + public String getSearchName() { + return this.searchName; + } + + /** + * {@return the search ID} + */ + public String getId() { + return this.id; + } + + /** + * {@return the list of search items belonging to the Search} + */ + public List getSearchItems() { + return Collections.unmodifiableList(searchItems); + } + + /** + * Set the description for the search. + * + * @param description of the search + */ + void setDescription(String description) { + this.description = description; + } + + /** + * {@return the description for the search} + */ + public String getDescription() { + return this.description; + } + + /** + * Set the contributor's identification. + * + * @param submitter who contributed the search + */ + void setSubmitter(String submitter) { + this.submitter = submitter; + } + + /** + * {@return the submitter's identification} + */ + public String getSubmitter() { + return this.submitter; + } + + /** + * Add attribute from the {@code} tag to the map for use downstream. + * + * @param key attribute key + * @param value attribute value + */ + void setAttribute(String key, String value) { + this.additionalAttributes.put(key, value); + } + + /** + * {@return the additional attributes parsed from the {@code} tag} + */ + public Map getAdditionalAttributes() { + return Collections.unmodifiableMap(additionalAttributes); + } + + @Override + public String toString() { + return this.searchName; + } + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchItem.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchItem.java new file mode 100644 index 0000000000..d5d4849a02 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchItem.java @@ -0,0 +1,172 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +import java.util.*; + +/** + * This class represents a parsed search item as used by the Byte Pattern Searcher capability within + * Ghidra. + *

    + * The design of the XML Parser and its supporting classes can be found in {@link BpsXmlParser}. + *

    + * The design of the Byte Pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This class manages the search item characteristics parsed from the {@code } XML tag. + * + *

    Byte Pattern Search Item Configurations

    + *

    + * Required Variables: + *

      + *
    • patternName – Name of the pattern for referencing.
      • + *
    • wordSize – The pattern word size, resolved from the WordSize attribute from the + * {@code} tag.
      • + *
    • searchType – The {@link BpsSearchType} strategy (table, constant, function, or ordered + * function search, see {@link BpsSearchType}) which dictates how patterns are searched.
      • + *
    • patterns – The {@code } collection of search byte sequences.
      • + *
    + * + *

    + * Optional Variables: + *

      + *
    • description – Text explanation detailing the purpose of the search pattern.
      • + *
    • submitter – The contributor who added this pattern to the {@link BpsSearch}.
      • + *
    + * Note: additional attributes may be included on the {@code} tag, they + * will be stored in a map for filtering and sorting downstream. + */ +public class BpsSearchItem { + + private String patternName; + private String id; + private int wordSize; + private BpsSearchType searchType; + private List patterns; + private String description; + private String submitter; + private Map additionalAttributes; + + /** + * Constructor. + * + * @param patternName name of pattern + * @param key unique key to facilitate skipping patterns at the level + * @param wordSize word size for pattern bytes (e.g. byte-8, word-16, dword-32, qword-64) + * @param searchType how the bytes should be searched + */ + public BpsSearchItem(String patternName, String key, int wordSize, BpsSearchType searchType) { + this.patternName = patternName; + this.id = key; + this.wordSize = wordSize; + this.searchType = searchType; + this.additionalAttributes = new HashMap<>(); + } + + /** + * {@return the pattern name} + */ + public String getPatternName() { + return this.patternName; + } + + /** + * {@return the search item's ID} + */ + public String getId() { + return this.id; + } + + /** + * {@return the pattern's word size} + */ + public int getWordSize() { + return this.wordSize; + } + + /** + * {@return the pattern's search type} + */ + public BpsSearchType getSearchType() { + return this.searchType; + } + + /** + * {@return the pattern's byte collection} + */ + public List getPatterns() { + return Collections.unmodifiableList(this.patterns); + } + + /** + * Set collection of byte arrays for the pattern. + * + * @param patterns the byte arrays representing the pattern + */ + void setBytePatterns(List patterns) { + this.patterns = new ArrayList<>(patterns); + } + + /** + * Set the search item's description. + * + * @param description of the search item + */ + void setDescription(String description) { + this.description = description; + } + + /** + * {@return the search item's description} + */ + public String getDescription() { + return this.description; + } + + /** + * Set pattern submitter identification. + * + * @param submitter of the search item + */ + void setSubmitter(String submitter) { + this.submitter = submitter; + } + + /** + * {@return the submitter of the pattern} + */ + public String getSubmitter() { + return this.submitter; + } + + /** + * Add attribute to the list for use downstream. + * + * @param key attribute key + * @param value attribute value + */ + void addAttribute(String key, String value) { + this.additionalAttributes.put(key, value); + } + + /** + * {@return the additional attributes map parsed from the {@code} tag} + */ + public Map getAdditionalAttributes() { + return Collections.unmodifiableMap(this.additionalAttributes); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchType.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchType.java new file mode 100644 index 0000000000..7a8a620d33 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsSearchType.java @@ -0,0 +1,54 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +/** + * This enum represents the type of search to be conducted as used by the Byte Pattern Searcher + * capability within Ghidra. + *

    + * The design of the Byte Pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This enum manages the possible supported search types. + *

    + * All {@link BpsSearchItem} objects will have a searchType which indicate how the + * {@link BpsPattern} items should be searched for. A complete discussion of all search types can be + * found in the "Search For Byte Patterns" page in the help viewer. + */ +public enum BpsSearchType { + /** + * Table Search: byte patterns represent a table and must be found in the same order + * as in the pattern file + */ + TABLE_SEARCH, + + /** + * Constant Search: byte patterns to be found anywhere and in any order in the program. + */ + CONSTANT_SEARCH, + + /** + * Function Search: byte patterns must all be found within the same function. + */ + FUNCTION_SEARCH, + + /** + * Ordered Function Search: byte patterns must all be found within the same function + * and also be found in the same order as in the pattern file. + */ + ORDERED_FUNCTION_SEARCH +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformation.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformation.java new file mode 100644 index 0000000000..202ad2a829 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformation.java @@ -0,0 +1,94 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch.bytepatternsearch; + +import java.util.ArrayList; +import java.util.List; + +import ghidra.program.model.lang.Endian; + +/** + * This class records a byte pattern's transformation in support of Ghidra's Byte Pattern Search + * capability. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * Indicates the structural state mutations which have been applied to a {@link BpsPattern}. + *

    + * NOTE: Once patterns are generated, they do not change (instead, when transforming a + * pattern, a new pattern is generated and has its own transformation record) so pattern + * transformation records are not modifiable. + * + * @param endianness of the pattern + * @param extensionSize word size the pattern was padded to (can be null) + * @param splitToSize word size the pattern was split to (can be null) + * @param splitPartIndex index of the split that this pattern represents (eg. index=1 indicates the + * first part of a split) (can be null) + * @param totalSplitParts total count of all parts of a split pattern (can be null) + */ +public record BpsTransformation( + Endian endianness, + Integer extensionSize, + Integer splitToSize, + Integer splitPartIndex, + Integer totalSplitParts) { + + /** + * Factory method for an unmodified baseline pattern. + *

    + * NOTE: All patterns prior to any transformations are safely assumed to be big endian. + * + * @return the transformation + */ + public static BpsTransformation empty() { + return new BpsTransformation(Endian.BIG, null, null, null, null); + } + + /** + * {@return Creates a copy of this record that uses the Little Endian setting} + */ + public BpsTransformation asLittleEndian() { + Integer padSize = extensionSize; + Integer splitSize = splitToSize; + Integer splitIndex = splitPartIndex; + Integer splitParts = totalSplitParts; + return new BpsTransformation(Endian.LITTLE, padSize, splitSize, splitIndex, + splitParts); + } + + @Override + public String toString() { + return getDescription(); + } + + /** + * {@return the description of the pattern's transformation history} + */ + public String getDescription() { + List steps = new ArrayList<>(); + steps.add("Endian: " + endianness.getDisplayName()); + if (extensionSize != null) { + steps.add("Padded to " + extensionSize); + } + if (splitPartIndex != null) { + steps.add(String.format("Split to %d Word Size | Chunk %d of %d", splitToSize, + splitPartIndex, totalSplitParts)); + } + return String.join(" -> ", steps); + } + +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformationEngine.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformationEngine.java new file mode 100644 index 0000000000..ce984158c4 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsTransformationEngine.java @@ -0,0 +1,412 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch.bytepatternsearch; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +import ghidra.program.model.lang.Endian; + +/** + * This class performs byte pattern transformations in support of Ghidra's Byte Pattern Search + * capability. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This transformer is used to prepare byte patterns parsed by the {@link BpsXmlParser} and + * populated in {@link BpsSearchItem} objects, prior to submitting them for search. + *

    + * Supported transformations are as follows: + *

      + *
    • Little Endian - Reverse a big Endian byte array. This is determined either + * programmatically based on the program's endianness or is user-controlled.
      • + *
    • Byte Splitting - If the pattern's word size is larger than the program's word + * boundary, split the pattern to fit the boundary. This is determined programmatically and is not + * user-controlled.
      • + *
    • Byte Zero-Padding - If the pattern's word size is smaller than the program's word + * boundary, zero-pad the byte pattern to fit the program's word boundary. The decision to extend + * bytes is user-controlled.
      • + *
    + *

    + * A skip list is used to allow users to skip patterns at the {@link BpsSearch} and + * {@link BpsSearchItem} level. + */ +public class BpsTransformationEngine { + + /** + * Process an entire search library parsed by {@link BpsXmlParser} and populated in + * {@link BpsSearch} and {@link BpsSearchItem} objects in preparation for search submission. + * + * @param searchLibrary collection of search objects + * @param config search preferences from user input and program details + * + * @return All patterns prepared for search submission + */ + public List createPatterns(List searchLibrary, BpsConfig config) { + return createPatterns(searchLibrary, null, config); + } + + /** + * Process an entire search library parsed by {@link BpsXmlParser} and populated in + * {@link BpsSearch} and {@link BpsSearchItem} objects in preparation for search submission. + *

    + * Skip over any patterns whose id's are included in a skip list. + * + * @param searchLibrary collection of search objects + * @param skipIds list of {@link BpsSearch} and/or {@link BpsSearchItem} id's to skip over + * @param config search preferences from user input and program details + * + * @return All patterns prepared for search submission + */ + public List createPatterns(List searchLibrary, + Set skipIds, BpsConfig config) { + + List results = new ArrayList<>(); + if (skipIds == null) { + skipIds = Collections.emptySet(); + } + + for (BpsSearch search : searchLibrary) { + if (skipIds.contains(search.getId())) { + continue; + } + for (BpsSearchItem searchItem : search.getSearchItems()) { + if (skipIds.contains(searchItem.getId())) { + continue; + } + + int patternWordSize = searchItem.getWordSize(); + List patterns = searchItem.getPatterns(); + List transformed = transformPatterns(patterns, config, patternWordSize); + results.addAll(transformed); + } + } + return results; + } + + /** + * Transform patterns to accommodate search configurations as identified in {@link BpsConfig}. + *

    + * Transformations are completed in the following order: + *

      + *
    1. Little Endian - generate the LE arrangement from the BE pattern.
      2. + *
    2. Byte Splitting - If the pattern's word size is larger than the program's word + * boundary, split the pattern to fit the boundary.
      3. + *
    3. Byte Zero-Padding - If the pattern's word size is smaller than the program's word + * boundary, zero-pad the byte pattern to fit the program's word boundary.
      4. + *
    + * Generally, these transformations are determined by the program's details (endianness and word + * boundary), but user input may also dictate how patterns are prepared for search. + *

    + * + * @param originals collection of byte patterns for transformation + * @param config search configuration + * @param searchItemWordSize the search item word size + * + * @return collection of transformed byte patterns + */ + private List transformPatterns(List originals, + BpsConfig config, int searchItemWordSize) { + + // 1. Process endianness if needed + // Endianness does not matter and splitting is not needed if word size is a single byte; + // keep constructor simple + boolean ignoreEndianess = searchItemWordSize <= 8; + List endianOutput = new ArrayList<>(); + if (!ignoreEndianess && config.isSearchLE()) { + for (BpsPattern pattern : originals) { + if (config.isSearchBE()) { + endianOutput.add(pattern); // maintain BE arrangement + } + endianOutput.add(createLEPattern(pattern)); + } + } + else { + endianOutput = originals; + } + + // 2. Process pattern splitting if needed + boolean splitPattern = config.getProgramWordSize() < searchItemWordSize; + List splitOutput = new ArrayList<>(); + if (splitPattern) { + for (BpsPattern pattern : endianOutput) { + splitOutput.addAll(createSplitPattern(pattern, config)); + } + } + else { + splitOutput = endianOutput; + } + + // 3. Process pattern expansion if needed + List finalOutput = new ArrayList<>(); + if (config.isExtendPattern()) { + for (BpsPattern pattern : splitOutput) { + finalOutput.addAll(createExtendedPattern(pattern, config)); + } + } + else { + finalOutput = splitOutput; + } + return finalOutput; + } + + /** + * Rearrange a big endian pattern to be little endian. This is generally completed prior to + * performing any zero-extending or splitting of byte patterns. + * + * @param pattern big endian byte pattern + * + * @return little endian pattern arrangement + */ + private BpsPattern createLEPattern(BpsPattern pattern) { + + BpsTransformation transformation = pattern.getTransformation(); + BpsTransformation leTransformation = transformation.asLittleEndian(); + + ByteBuffer leBuff = createLitteEndianBuffer(pattern); + String name = pattern.getName(); + BpsSearchItem searchItem = pattern.getSearchItem(); + return new BpsPattern(name, leBuff, searchItem, leTransformation); + } + + /** + * Generate LE arrangement of pattern. + * + * @param pattern to make LE + * + * @return pattern LE byte buffer + */ + private ByteBuffer createLitteEndianBuffer(BpsPattern pattern) { + ByteBuffer originalBuffer = pattern.getBuffer(); + int capacity = originalBuffer.capacity(); + ByteBuffer leBuff = ByteBuffer.allocate(capacity); + leBuff.order(ByteOrder.LITTLE_ENDIAN); + + BpsSearchItem searchItem = pattern.getSearchItem(); + int wordSize = searchItem.getWordSize(); + switch (wordSize) { + case 16: + short asShort = originalBuffer.getShort(); + leBuff.putShort(asShort); + break; + + case 32: + int asInt = originalBuffer.getInt(); + leBuff.putInt(asInt); + break; + + case 64: + long asLong = originalBuffer.getLong(); + leBuff.putLong(asLong); + break; + } + return leBuff; + } + + /** + * Split a byte pattern to fit within the program's word boundary as indicated in the search + * configuration {@link BpsConfig}. More than a single split is possible depending on the word + * size of the pattern and the word size boundary of the program. + *

    + * All split byte segments are placed in a list and stored as separate, split, patterns as + * tracked by each pattern's {@link BpsTransformation}. The searcher will submit searches and + * perform follow-on processing to ensure that split patterns are found appropriately adjacent + * to each other. + *

    + * This transformation is generally performed after little endian rearrangement, if it is + * required, and prior to pattern zero-extention, if it is required. + * + * @param pattern to be split + * @param config search configuration + * + * @return split pattern + */ + private List createSplitPattern(BpsPattern pattern, BpsConfig config) { + + // Pattern's byte buffer has read-only protections + ByteBuffer originalByteBuffer = pattern.getBuffer(); + byte[] bytes = new byte[originalByteBuffer.remaining()]; + + BpsSearchItem searchItem = pattern.getSearchItem(); + int programWordSize = config.getProgramWordSize(); + int numOfSplits = searchItem.getWordSize() / programWordSize; + int targetByteSize = bytes.length / numOfSplits; + byte[][] splitBytes = new byte[numOfSplits][targetByteSize]; + + List splitPatterns = new ArrayList(); + for (int i = 0; i < numOfSplits; i++) { + originalByteBuffer.get(splitBytes[i], 0, targetByteSize); + BpsTransformation transformation = pattern.getTransformation(); + Endian endianness = transformation.endianness(); + int splitIndex = i + 1; + + // If a pattern needs split, we can confidently assume that it has not been previously + // zero-extended, so we can safely null out this value in the new record. + BpsTransformation newTransformation = new BpsTransformation(endianness, + null, programWordSize, splitIndex, numOfSplits); + + String patternName = pattern.getName(); + ByteBuffer buffer = ByteBuffer.wrap(splitBytes[i]); + BpsPattern splitPattern = + new BpsPattern(patternName, buffer, searchItem, newTransformation); + + splitPatterns.add(splitPattern); + } + return splitPatterns; + } + + /** + * Zero-extend a byte pattern to fill the program's word size boundary. We perform padding + * quantization to accommodate program word sizes which are more than twice the size of the + * pattern's word size. + *

    + * Quantization Extension Examples: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    Pattern Word SizeProgram Word SizeGenerated Extended Patterns
    166432 and 64
    326464
    + *

    + * This transformation is generally performed after little endian rearrangement, if it is + * required, and after pattern splitting, if it is required. + * + * @param pattern byte pattern for extension + * @param config search configuration + * + * @return extended pattern + */ + private List createExtendedPattern(BpsPattern pattern, BpsConfig config) { + + List paddedBytesList = new ArrayList(); + BpsSearchItem searchItem = pattern.getSearchItem(); + int searchItemWordSize = searchItem.getWordSize(); + int programWordSize = config.getProgramWordSize(); + List padSizes = getPadSizes(searchItemWordSize, programWordSize); + + for (int size : padSizes) { + BpsPattern extendedPattern = padByteArray(pattern, size); + paddedBytesList.add(extendedPattern); + } + return paddedBytesList; + } + + /** + * If the user requests to extend the data types to fit in the program's word size (and the + * search bytes' word size is smaller than the program's), we zero-pad the byte array. + *

    + * The endianness of the pattern dictates where the zeros are placed: + *

    + * Big endian: zeros are placed on the right of each byte - at the highest address + *

    + * Little endian: zeros are placed on the left of each byte - at the lowest address + *

    + * + * @param pattern to be extended + * @param targetSize word size to extend to + * + * @return extended pattern + */ + private BpsPattern padByteArray(BpsPattern pattern, int targetSize) { + + // Calculate total bytes needed for the target size + int byteWordSize = 8; + int totalBytes = targetSize / byteWordSize; + ByteBuffer paddedBuffer = ByteBuffer.allocate(totalBytes); + + ByteBuffer storedBuffer = pattern.getBuffer().duplicate(); + byte[] bytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(bytes); + + // Insert zeros and data based on true Endian alignment + BpsTransformation transformations = pattern.getTransformation(); + Endian endianness = transformations.endianness(); + if (endianness == Endian.LITTLE) { + // Little Endian: Data goes at the lowest addresses (start of buffer) + paddedBuffer.put(bytes); + while (paddedBuffer.hasRemaining()) { + paddedBuffer.put((byte) 0x00); + } + } + else { + // Big Endian: Data goes at the highest addresses (end of buffer) + int numZeros = totalBytes - bytes.length; + for (int p = 0; p < numZeros; p++) { + paddedBuffer.put((byte) 0x00); + } + paddedBuffer.put(bytes); + } + + // Expanded patterns will not previously have been split, so we can confidently null those + // details related to splitting out + BpsTransformation newTransformation = new BpsTransformation(endianness, targetSize, + null, null, null); + + String patternName = pattern.getName(); + BpsSearchItem parentSearchItem = pattern.getSearchItem(); + return new BpsPattern(patternName, paddedBuffer, parentSearchItem, newTransformation); + } + + /** + * Helper method for determining the collection of word sizes to zero-pad a byte array to. + *

    + * This could be 1 or more size values depending on the comparison between the pattern's word + * size and the program's word size. + *

    + * Quantization Example: if the pattern word size is 16bit and the program's word size is + * 64, we should zero-pad to 32bit and 64bit - returning both of these numbers as pad sizes to + * cycle through later. + *

    + * NOTE: At this point, we know that zero-padding needs to take place based on previous checks. + * We are safely assuming that the word size of the search byte array is at least 16bit because + * of previous filtering. + *

    + * + * @param patternWordSize bit length of the pattern to extend + * @param progWordSize bit length of the program's word size boundary + * + * @return list of word sizes to extend to + */ + private List getPadSizes(int patternWordSize, int progWordSize) { + List padSizes = new ArrayList(); + + switch (progWordSize) { + case 32: + padSizes.add(32); + break; + case 64: + // Quantization: if the byte array's word size is 16, we pad to 32 as well as 64. + if (patternWordSize == 16) { + padSizes.add(32); + } + padSizes.add(64); + } + return padSizes; + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsXmlParser.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsXmlParser.java new file mode 100644 index 0000000000..2f405714e0 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/bytepatternsearch/BpsXmlParser.java @@ -0,0 +1,639 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.Strings; +import org.xml.sax.*; + +import ghidra.app.util.importer.MessageLog; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.TaskMonitor; +import ghidra.xml.*; + +/** + * This is the main class which parses pattern search XML files as used by the Byte Pattern Search + * capability within Ghidra. + *

    + * The design of the Byte pattern Searcher and its supporting sub-capabilities and classes can be + * found in the "Search For Byte Patterns" page in the help viewer. + *

    + * This class parses the provided byte pattern search XML files, populating corresponding Byte + * Pattern Searcher objects: {@link BpsSearch},{@link BpsSearchItem}, and {@link BpsPattern} for + * processing and searching downstream. + *

    + * A search pattern XML file contains a set of byte patterns, organized into a simple hierarchy, + * which contain specific meta-data details that will shape the search criteria. A full discussion + * on the required XML schema can be found in the "Search For Byte Patterns" page in the help + * viewer. + *

    + * Note: All XML search documents must adhere to the structure outlined in the Byte + * Pattern Search help file. Parse errors due to malformed XML or an XML structure that does not + * follow the form, will result in accumulated parse errors which will be displayed once parsing has + * completed. + */ +public class BpsXmlParser { + + private static final String BPS_LIBRARY = "library"; + private static final String NAME = "Name"; + private static final String SEARCH_TYPE = "SearchType"; + private static final String DATA_TYPE = "DataType"; + private static final String WORD_SIZE = "WordSize"; + private static final String SEARCH_ITEM = "SearchItem"; + private static int keyCounter; // Used for unique key generation within a pattern file + + /*** + * Parse the provided XML file, generating a list of {@link BpsSearch} objects. A map is used to + * support the skipping of patterns at the {@code} and {@code} levels. + *

    + * All pattern files must have surrounding {@code} tags and adhere to + * the structure outlined in the Byte pattern Search help file. + *

    + * + * @param searchFile XML file to be parsed + * @param monitor the task monitor + * + * @return Byte pattern searches + * + * @throws SAXException for malformed XML parse errors + */ + public static List parseSearchFile(File searchFile, + TaskMonitor monitor) throws SAXException { + + try { + return doParseSearchFile(searchFile, monitor); + } + catch (CancelledException e) { + return List.of(); + } + } + + private static List doParseSearchFile(File searchFile, + TaskMonitor monitor) throws SAXException, CancelledException { + + XmlMessageLog log = new XmlMessageLog(); + AccumulatingErrorHandler errorHandler = new AccumulatingErrorHandler(log); + XmlPullParser parser; + keyCounter = 0; // Reset the key for each pattern file + List searches = new ArrayList<>(); + + try { + parser = XmlPullParserFactory.create(searchFile, errorHandler, false); + + String parseName = parser.peek().getName().toLowerCase(); + if (!parseName.contains(BPS_LIBRARY)) { + trackParseError( + "Line: " + parser.getLineNumber() + ", Col: " + parser.getColumnNumber() + + ": Pattern file must have a wrapper tag " + + "to start.", + log, errorHandler); + } + + XmlElement startSearchElement = parser.start(); + + // Each pattern file contains a collection of searches + while (parser.peek().isStart()) { + + monitor.checkCancelled(); + + String tag = parser.peek().getName(); + // This new version of Strings will ignore case by default + if (Strings.CS.containsAny(tag, "Search", "Algorithm", "Family")) { + List search = parseSearch(parser, log, errorHandler, monitor); + searches.addAll(search); + } + else { + trackParseError( + "Line: " + parser.getLineNumber() + ", Col: " + parser.getColumnNumber() + + ": tag expected, current tag not supported: " + tag + ".", + log, errorHandler); + parser.discardSubTree(tag); + } + } + + parser.end(startSearchElement); + } + catch (SAXException | IOException e) { + log.appendException(e); + // Some severe fatal errors will break the parsing early, but standard violation + // errors will be collected for review after parsing completes. + } + + // Check if any errors were collected during the parsing process + if (errorHandler.foundErrors()) { + String combinedMessage = errorHandler.getExceptions() + .stream() + .map(exception -> exception.getMessage()) + .collect(Collectors.joining("\n")); + + // SAXParser exceptions do not allow for the inspection of each error encountered. + // Since we are collecting errors as we go, allowing the parser to continue until + // finished, we need more than a string message of the exception encountered - we need + // access to the underlying exceptions for precise testing and error resolution. + throw new BpsXmlValidationException( + "XML parsing failed with " + errorHandler.getExceptions().size() + " errors:\n" + + combinedMessage, + errorHandler.getExceptions()); + } + log.appendMsg("XML parsed successfully with zero errors"); + + return searches; + } + + /** + * Parse a {@code} tag and subsequent {@code } tags. Searches are stored as + * {@link BpsSearch} objects and have 1 required tag attribute: "Name". + *

    + * + * @param parser the parser + * @param log message log + * @param errorHandler for standard violation XML errors + * @param monitor the task monitor + * + * @return the search items + * + * @throws SAXException for malformed XML parse errors + */ + private static List parseSearch(XmlPullParser parser, XmlMessageLog log, + AccumulatingErrorHandler errorHandler, TaskMonitor monitor) + throws SAXException, CancelledException { + + XmlElement searchStart = parser.start("Family", "Search"); + List searches = new ArrayList<>(); + + // Search tags must have a "Name" attribute + if (searchStart.hasAttribute(NAME)) { + + String name = searchStart.getAttribute(NAME); + // Only 1 attribute to skip, gatherTagAttributes() requires this to be a list + List skipAttribute = List.of(NAME); + Map additionalAttributes = + gatherTagAttributes(searchStart, skipAttribute); + + List searchItems = new ArrayList<>(); + while (parser.peek().isStart()) { + + monitor.checkCancelled(); + + String tag = parser.peek().getName(); + if (!SEARCH_ITEM.equals(tag)) { + trackParseError("Line: " + parser.getLineNumber() + ", Col: " + + parser.getColumnNumber() + ": Tag not supported: " + tag + + ", tag expected. Skipping subtree.", log, errorHandler); + parser.discardSubTree(parser.next()); // Un-handled element + continue; + } + + BpsSearchItem item = parseSearchItem(parser, log, errorHandler); + if (item != null) { + searchItems.add(item); + } + } + + parser.end(searchStart); + + if (!searchItems.isEmpty()) { + + String parserName = parser.getName(); + String key = generateKey(parserName, name); + BpsSearch search = new BpsSearch(name, key, searchItems); + populateSearchAttributes(additionalAttributes, search, log); + searches.add(search); + } + else { + log.appendMsg("No valid found for family: " + name + ", skipping."); + } + + } + else { + trackParseError("Line: " + parser.getLineNumber() + ", Col: " + + parser.getColumnNumber() + + ": All tags must contain a 'Name' attribute. Skipping this Search.", + log, errorHandler); + parser.discardSubTree(searchStart); + } + + return searches; + } + + /** + * The key helps differentiate between searches to support skipping patterns at the + * {@code} and {@code} levels. + *

    + * Example key: filename.xml_SearchName_1 + * + * @param fileName pattern file name + * @param name parsed {@code} or {@code} name + * + * @return generated key + */ + private static String generateKey(String fileName, String name) { + return (String.format("%s_%s_%d", fileName, name, keyCounter++)); + } + + /** + * Unroll the mapping of key/value attribute pairs parsed from the XML {@code } tag and + * populate attribute values in {@link BpsSearch}. + *

    + * NOTE: this is kept separate from buildSearchItem because there are some attributes between + * the two tags ({@code} and {@code}) which overlap in name, but not in + * value. Specifically, both tags may contain "Description" and "Submitter" keys but their + * values will likely be different and need to be maintained separately. Keeping these separate + * requires two different managers to populate their corresponding object values. + *

    + * + * @param additionalTagAttributes extra parsed attributes on the {@code } tag + * @param search object that the tag attributes are assigned to + * @param log the XML log + */ + private static void populateSearchAttributes( + Map additionalTagAttributes, BpsSearch search, + XmlMessageLog log) { + + for (String key : additionalTagAttributes.keySet()) { + String value = additionalTagAttributes.get(key); + switch (key) { + case "Description": + search.setDescription(value); + break; + + case "Submitter": + search.setSubmitter(value); + break; + + default: + log.appendMsg( + "Unknown attribute on tag encountered: " + key + ", adding to " + + "map for downstream use."); + search.setAttribute(key, value); + break; + } + } + } + + /** + * Parse a {@code} tag and subsequent {@code} tags. SearchItems are stored as + * {@link BpsSearchItem} objects and have 3 required tag attributes: "Name", "WordSize", and + * "SearchType". + * + * @param parser the parser + * @param log the XML log + * @param errorHandler for handling standard violation XML errors + * + * @throws SAXException for malformed XML parse errors + */ + private static BpsSearchItem parseSearchItem(XmlPullParser parser, + XmlMessageLog log, AccumulatingErrorHandler errorHandler) throws SAXException { + + XmlElement searchTag = parser.start("SearchItem"); + + // Gather required attributes. "DataType" is a compatible substitute for WordSize + if (!searchTag.hasAttribute(NAME) || !searchTag.hasAttribute(SEARCH_TYPE) || + (!searchTag.hasAttribute(WORD_SIZE) && !searchTag.hasAttribute(DATA_TYPE))) { + trackParseError("Line: " + parser.getLineNumber() + ", Col: " + + parser.getColumnNumber() + + ": All tags must contain: 'Name', " + + "'WordSize', and 'SearchType' attributes. Skipping this SearchItem.", log, + errorHandler); + parser.discardSubTree(searchTag); + return null; + } + + String name = searchTag.getAttribute(NAME); + + // "DataType" is a compatible substitute for WordSize + String targetValue = + (searchTag.getAttribute(WORD_SIZE) != null) ? searchTag.getAttribute(WORD_SIZE) + : searchTag.getAttribute(DATA_TYPE); + int wordSize = lookUpWordSize(targetValue, log, errorHandler); + + String searchType = searchTag.getAttribute(SEARCH_TYPE); + List attributeSkipList = List.of(NAME, WORD_SIZE, SEARCH_TYPE); + + Map additionalAttributes = + gatherTagAttributes(searchTag, attributeSkipList); + + String fileName = parser.getName(); + BpsSearchItem searchItem = buildSearchItem(name, fileName, wordSize, searchType, + additionalAttributes, log, errorHandler); + + List byteCollection = + gatherSearchItemBytes(parser, wordSize, searchItem, log, errorHandler); + + if (byteCollection != null) { + searchItem.setBytePatterns(byteCollection); + } + else { + log.appendMsg( + "No byte pattern found for SearchItem: " + name + " line: " + + parser.getLineNumber() + ", skipping."); + } + parser.end(searchTag); + + return searchItem; + } + + /** + * Generate {@link BpsSearchItem} object from parsed {@code} tag. + *

    + * + * @param name SearchItem name + * @param wordSize byte pattern word size + * @param parsedSearchType the search type (e.g. function, constant, table, or orderedFunction) + * default is constant search + * @param additionalAttributes parsed extra attributes on the {@code} tag + * @param log the XML log + * @param errorHandler for handling invalid "searchType" attribute value + * + * @return the search item + * + * @throws SAXException for malformed XML parse errors + */ + private static BpsSearchItem buildSearchItem(String name, String filename, int wordSize, + String parsedSearchType, + Map additionalAttributes, XmlMessageLog log, + AccumulatingErrorHandler errorHandler) throws SAXException { + + BpsSearchType searchType; + switch (parsedSearchType.toLowerCase()) { + case "function": + searchType = BpsSearchType.FUNCTION_SEARCH; + break; + case "constant": + searchType = BpsSearchType.CONSTANT_SEARCH; + break; + case "table", "wholetable": + searchType = BpsSearchType.TABLE_SEARCH; + break; + case "orderedfunction": + searchType = BpsSearchType.ORDERED_FUNCTION_SEARCH; + break; + default: + trackParseError("SearchType: '" + parsedSearchType + "' is unknown. " + + "Using default constant search.", log, errorHandler); + + searchType = BpsSearchType.CONSTANT_SEARCH; + break; + } + + String generatedKey = generateKey(filename, name); + BpsSearchItem searchItem = + new BpsSearchItem(name, generatedKey, wordSize, searchType); + + for (String key : additionalAttributes.keySet()) { + String value = additionalAttributes.get(key); + switch (key) { + case "Description": + searchItem.setDescription(value); + break; + case "Submitter": + searchItem.setSubmitter(value); + break; + default: + log.appendMsg( + "Unknown attribute on tag encountered: " + key + + ", stored for use " + + "downstream."); + searchItem.addAttribute(key, value); + break; + } + } + return searchItem; + } + + /** + * Parse each {@code } sub tag. Bytes are currently assumed to be Hex or ASCII values and + * are parsed & stored accordingly. ASCII values are indicated in the XML as having a word size + * = -1. Each {@code} has at least 1 {@code} tag but could have hundreds of + * separate ones. + *

    + * This method parses all {@code} tags and returns the list of {@link BpsPattern} objects + * which contain, not just the parsed byte string, but also metadata about the bytes. + *

    + * NOTE: All byte patterns must be in big endian arrangement upon parsing. + * + * @param parser the parser + * @param wordSize the byte pattern word size + * @param log XML log + * @param errorHandler for standard XML violation errors + * + * @return list of byte patterns in the {@code} + * + * @throws SAXException for malformed hex errors + */ + private static List gatherSearchItemBytes(XmlPullParser parser, int wordSize, + BpsSearchItem parentSearchItem, XmlMessageLog log, + AccumulatingErrorHandler errorHandler) throws SAXException { + + List byteCollection = new ArrayList<>(); + + while (parser.peek().isStart()) { + XmlElement byteTag = parser.start(); + String tagName = byteTag.getName(); + + // Ghidra's XML parser ties text content to XML elements (unlike other standard parsers) + // so we must process the closing tag first in order to capture the text as that is + // the tag that the text is attached to. + byteTag = parser.end(byteTag); + String byteString = byteTag.getText().trim(); + byteString = byteString.replaceAll("\\s", ""); + + byte[] rawBytes; + ByteBuffer buffer = null; + if (wordSize == -1) { // indicates an ASCII value, handled differently + rawBytes = byteString.getBytes(); + buffer = ByteBuffer.wrap(rawBytes); + } + else if (byteString.length() % 2 == 0) { // well-formed hex + rawBytes = HexFormat.of().parseHex(byteString); + buffer = ByteBuffer.wrap(rawBytes); + + } + else { + trackParseError("Line: " + parser.getLineNumber() + ", Col: " + + parser.getColumnNumber() + ": Byte string is not an even size and " + + "cannot be parsed as Hex. Either adjust WordSize attribute to " + + "ASCII or zero-pad hex to make it well-formed. Skipping entire " + + "search item.", log, errorHandler); + + // If even 1 pattern is invalid, skip all tags in the + // So that we don't perform a search on an incomplete + while (parser.peek().getName().equals("Bytes")) { + parser.next(); + } + return null; // force the skipping of this + } + byteCollection.add( + new BpsPattern(tagName, buffer, parentSearchItem, BpsTransformation.empty())); + } + return byteCollection; + } + + /** + * Parse attribute values in the form of "Name"="Value". Any number of attributes may be + * included in the tag. Any attributes not outlined as required or optional will still be kept + * in a map for filtering downstream. + *

    + * NOTE: All required attributes for a particular tag must be handled separately and are + * included in the skipList parameter so that they are not processed twice. + *

    + * + * @param currTag current XML element + * @param skipList required attributes to skip and not be parsed/stored twice + * + * @return gathered tag attributes + */ + private static Map gatherTagAttributes(XmlElement currTag, + List skipList) { + + Iterator> attributes = currTag.getAttributeIterator(); + Map additionalTags = new TreeMap<>(); + + while (attributes.hasNext()) { + Entry attribute = attributes.next(); + String key = attribute.getKey(); + if (!skipList.contains(key)) { + additionalTags.put(key, attribute.getValue()); + } + } + return additionalTags; + } + + /** + * Word sizes in the XML are indicated as String word sizes. Translate these to int values for + * use later. ASCII is encoded with a value of -1 and an unknown data type is encoded as the + * default size of 16 (word) with an error message appended to the handler. + *

    + * + * @param wordSize parsed from "WordSize" attribute on {@code} tag + * @param errorHandler track any errors encountered with parsing the word size + * @param log for logging progress + * + * @return word size, default 16 indicates an invalid word size + * + * @throws SAXException for invalid word size + */ + private static int lookUpWordSize(String wordSize, XmlMessageLog log, + AccumulatingErrorHandler errorHandler) throws SAXException { + switch (wordSize.toLowerCase()) { + case "byte": + return 8; + case "word": + return 16; + case "dword": + return 32; + case "qword": + return 64; + case "ascii": + return -1; + default: + trackParseError("Invalid word size encountered: " + wordSize + ". Default size " + + "of 16 is being used.", log, errorHandler); + return 16; + } + } + + /** + * Helper for generating error alerts during parsing. + *

    + * + * @param specificMessage error message specific to the problem encountered. + * @param log for tracking progress + * @param errorHandler for handling standard XML violation errors + * + * @throws SAXException for handling malformed XML parse errors + */ + private static void trackParseError(String specificMessage, + XmlMessageLog log, AccumulatingErrorHandler errorHandler) throws SAXException { + log.appendMsg(specificMessage); + errorHandler.error(new SAXParseException(specificMessage, null)); + } +} + +/** + * Custom exception class adds a field for holding collected errors for ease of access and testing + * downstream. + */ +class BpsXmlValidationException extends SAXException { + private final List errors = new ArrayList<>(); + + BpsXmlValidationException(String message, List errors) { + super(message); + this.errors.addAll(errors); + } + + // This allows us to test the number of errors caught directly + int getErrorCount() { + return errors.size(); + } + + // Let users inspect the raw errors if needed + List getErrors() { + return errors; + } +} + +/** + * Custom ErrorHandler for XML parsing which keeps track of errors allowing parsing to complete + * instead of breaking on minor infractions in the XML. + */ +class AccumulatingErrorHandler implements ErrorHandler { + private MessageLog log; + private final List exceptions = new ArrayList<>(); + + AccumulatingErrorHandler(MessageLog log) { + this.log = log; + } + + @Override + public void error(SAXParseException exception) throws SAXException { + log.appendMsg(exception.getMessage()); + exceptions.add(exception); + } + + @Override + public void fatalError(SAXParseException exception) throws SAXException { + log.appendMsg(exception.getMessage()); + exceptions.add(exception); + } + + @Override + public void warning(SAXParseException exception) throws SAXException { + log.appendMsg(exception.getMessage()); + exceptions.add(exception); + } + + /** + * {@return the list of exceptions found during parsing} + */ + public List getExceptions() { + return exceptions; + } + + /** + * {@return the error status of the parser} + */ + public boolean foundErrors() { + return !exceptions.isEmpty(); + } +} diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsByteManipulationTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsByteManipulationTest.java new file mode 100644 index 0000000000..5f8d1e1ddc --- /dev/null +++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsByteManipulationTest.java @@ -0,0 +1,1274 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch.bytepatternsearch; + +import static org.junit.Assert.*; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.*; + +import org.junit.Test; +import org.xml.sax.SAXException; + +import ghidra.program.model.lang.Endian; +import ghidra.util.task.TaskMonitor; +import resources.ResourceManager; + +public class BpsByteManipulationTest { + + BpsTransformationEngine patternTransformer = new BpsTransformationEngine(); + + /** + * Helper method to produce a list of {@link BpsPattern} objects for testing. + * + * @return patterns generated from provided byte string. + */ + private List createPatterns(String bytes, int wordSize) { + byte[] rawBytes = HexFormat.of().parseHex(bytes); + ByteBuffer buffer = ByteBuffer.wrap(rawBytes); + + BpsPattern searchBytes = new BpsPattern("Bytes", buffer, + new BpsSearchItem("ParentSearchItem", "TestFile_TestItem_0", wordSize, + BpsSearchType.TABLE_SEARCH), + BpsTransformation.empty()); + return List.of(searchBytes); + } + + /** + * Helper method to generate wrappers around the provided search item so that + * .prepareSearchItemBytes() can be called. + * + * @param searchItem BpsSearchItem + * + * @return list of searches + */ + private List makeSearchCollection(BpsSearchItem searchItem) { + List searchCollection = new ArrayList(); + List items = new ArrayList(); + items.add(searchItem); + + BpsSearch searchItems = new BpsSearch("SearchItems1", "key", items); + searchCollection.add(searchItems); + return searchCollection; + } + + /** + * Helper for search context preferences - big endian, no extension + * + * @param programWordSize word size of program + * + * @return search config + */ + private BpsConfig beSearch(int programWordSize, boolean isExtend) { + return new BpsConfigBuilder(programWordSize) + .setPerformExtension(isExtend) + .setBigEndianSearch(true) + .setLittleEndianSearch(false) + .build(); + } + + /** + * Helper for search context preferences - little endian, no extension + * + * @param programWordSize word size of program + * + * @return search config + */ + private BpsConfig leSearch(int programWordSize, boolean isExtend) { + return new BpsConfigBuilder(programWordSize) + .setPerformExtension(isExtend) + .setBigEndianSearch(false) + .setLittleEndianSearch(true) + .build(); + } + + /** + * Helper for search context preferences - big & little endian, no extension + * + * @param programWordSize word size of program + * + * @return search config + */ + private BpsConfig beLeSearch(int programWordSize, boolean isExtend) { + return new BpsConfigBuilder(programWordSize) + .setPerformExtension(isExtend) + .setBigEndianSearch(true) + .setLittleEndianSearch(true) + .build(); + } + + @Test + public void testNoTransformationsNeeded() { + // assume bytes have already been parsed + String bytes = "0123"; + List bytePatterns = createPatterns(bytes, 32); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key1", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // Prepare byte string based on search preferences indicated in the configObject + // these would be determined at runtime + BpsConfig config = beSearch(32, false); + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals(1, preppedPatterns.size()); + } + + /** + * Simple test to reverse big endian bytes parsed from XML file to little endian. Perform word + * data type. + */ + @Test + public void testLittleEndianByteArrangmentWord() { + + // assume bytes have already been parsed + String bytes = "0123"; + List bytePatterns = createPatterns(bytes, 16); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key1", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // Prepare byte string based on search preferences indicated in the configObject + // these would be determined at runtime + BpsConfig config = leSearch(32, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String littleEndian = "2301"; + + ByteBuffer byteBuffer = preppedPatterns.get(0).getBuffer(); + byte[] byteArray = new byte[byteBuffer.remaining()]; + byteBuffer.get(byteArray); + + assertEquals("Search bytes need reversed to match little endian machine", littleEndian, + HexFormat.of().formatHex(byteArray)); + } + + /** + * Simple test to reverse big endian bytes parsed from XML file to little endian. Perform DWord + * data type. + */ + @Test + public void testLittletEndianByteArrangementDWord() { + + List bytePatterns = createPatterns("01234567", 32); + BpsSearchItem searchItem = + new BpsSearchItem("Test Pattern", "key", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = leSearch(32, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals(1, preppedPatterns.size()); + + String littleEndian = "67452301"; + + ByteBuffer storedBytes = preppedPatterns.get(0).getBuffer(); + byte[] byteArray = new byte[storedBytes.remaining()]; + storedBytes.get(byteArray); + + assertEquals("Search bytes need reversed to match little endian machine", littleEndian, + HexFormat.of().formatHex(byteArray)); + } + + /** + * Simple test to reverse big endian bytes parsed from XML file to little endian. Perform QWord + * data type. + */ + @Test + public void testLittleEndianByteArrangementQWord() { + // QWord + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = leSearch(64, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String littleEndian = "efcdab8967452301"; + ByteBuffer storedBytes = preppedPatterns.get(0).getBuffer(); + byte[] byteArray = new byte[storedBytes.remaining()]; + storedBytes.get(byteArray); + + assertEquals("Search bytes need reversed to match little endian machine", littleEndian, + HexFormat.of().formatHex(byteArray)); + } + + /** + * The user has selected "little endian" in the GUI as an alternative search arrangement to the + * default. In this case, the original big endian arrangement should be kept with an additional + * arrangement for little endian made. + */ + @Test + public void testUserConfigLittleAndBigEndian() { + + // assume bytes have already been parsed + List bytePatterns = createPatterns("01234567", 32); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + BpsConfig config = beLeSearch(32, false); + + // prepare byte string based on search preferences indicated in the configObject + List preppedPatterns = patternTransformer.createPatterns(search, config); + + // since the user selected to search LE, assume BE should also be searched unless + // ignoreDefaultEndianness was checked + assertEquals(2, preppedPatterns.size()); + + String littleEndian = "67452301"; + ByteBuffer storedBytes = preppedPatterns.get(0).getBuffer(); + byte[] defaultSearchBytes = new byte[storedBytes.remaining()]; + storedBytes.get(defaultSearchBytes); + + storedBytes = preppedPatterns.get(1).getBuffer(); + byte[] littleEndianBytes = new byte[storedBytes.remaining()]; + storedBytes.get(littleEndianBytes); + + assertEquals("BE search needs kept", "01234567", + HexFormat.of().formatHex(defaultSearchBytes)); + assertEquals("Search bytes need reversed to match little endian machine", littleEndian, + HexFormat.of().formatHex(littleEndianBytes)); + } + + /** + * Simple test to ensure correct splitting of bytes between byte arrays when the DataType size + * is larger than the program size. + * + * If the original byte array needs to be split based on program size, the splits are placed in + * the re-arranged byte arrays and the original byte array should not be cleared out. + */ + @Test + public void testBigEndianByteArraySingleSplit() { + + // assume bytes have already been parsed + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + // DataType=QWord, SearchType="Constant", parsed bytes + BpsSearchItem searchItem = new BpsSearchItem("", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = beSearch(32, false); + + // prepare byte patterns based on search preferences indicated in the configObject + List preppedPatterns = patternTransformer.createPatterns(search, config); + + String firstSplit = "01234567"; + String secondSplit = "89abcdef"; + // split byte arrays + + BpsPattern bpsPatternFirstSplit = preppedPatterns.get(0); + ByteBuffer storedBuffer = bpsPatternFirstSplit.getBuffer(); + byte[] firstSplitBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(firstSplitBytes); + + BpsPattern bpsPatternSecondSplit = preppedPatterns.get(1); + storedBuffer = bpsPatternSecondSplit.getBuffer(); + byte[] secondSplitBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(secondSplitBytes); + + // evaluate the bytes for each split + assertEquals("Original pattern split twice", 2, preppedPatterns.size()); + assertEquals("QWord needs to split to accommodate 32bit prog size", firstSplit, + HexFormat.of().formatHex(firstSplitBytes)); + assertEquals("QWord needs split to accommodate 32bit prog size", secondSplit, + HexFormat.of().formatHex(secondSplitBytes)); + + // evaluate the description for each split + BpsTransformation firstSplitTransformation = bpsPatternFirstSplit.getTransformation(); + Integer firstSplitPartIndex = firstSplitTransformation.splitPartIndex(); + assertEquals("This is the first split", 1, firstSplitPartIndex.intValue()); + assertEquals("Description needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 64] -> Pattern: 01 23 45 67 | Form: " + + "[Endian: Big -> Split to 32 Word Size | Chunk 1 of 2]", + bpsPatternFirstSplit.getDescription()); + + BpsTransformation secondSplitTransformation = bpsPatternSecondSplit.getTransformation(); + Integer secondSplitPartIndex = secondSplitTransformation.splitPartIndex(); + assertEquals("This is the second split", 2, secondSplitPartIndex.intValue()); + assertEquals("Description needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 64] -> Pattern: 89 AB CD EF | Form: " + + "[Endian: Big -> Split to 32 Word Size | Chunk 2 of 2]", + bpsPatternSecondSplit.getDescription()); + } + + /** + * Make sure splitting is happening generically, adjust search data type to DWord and program + * size to 16. + */ + @Test + public void testBigEndianSplitDWord() { + + List bytePatterns = createPatterns("01234567", 32); + + BpsSearchItem searchItem = + new BpsSearchItem("Search 1", "key", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + BpsConfig config = beSearch(16, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String firstSplit = "0123"; + String secondSplit = "4567"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + assertEquals("DWord needs split to accommodate 16bit prog size", firstSplit, + HexFormat.of().formatHex(storedBytes1)); + + assertEquals("DWord needs split to accommodate 16bit prog size", secondSplit, + HexFormat.of().formatHex(storedBytes2)); + } + + /** + * Ensure correct splitting of bytes between byte arrays when the DataType size is larger than + * the program size -- in this case, multiple splits of the byte array will be needed. + */ + @Test + public void testBigEndianMultiSplit() { + + // assume bytes have already been parsed, QWord + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = beSearch(16, false); // this will require 4 splits of the byte array + + // prepare byte string based on search preferences indicated in the configObject + List preppedPatterns = patternTransformer.createPatterns(search, config); + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + assertEquals("There should be 4 splits, do not keep the original byte array", 4, + preppedPatterns.size()); + + String firstSplit = "0123"; + String secondSplit = "4567"; + String thirdSplit = "89ab"; + String fourthSplit = "cdef"; + + storedBuffer = preppedPatterns.get(0).getBuffer(); + storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + storedBuffer = preppedPatterns.get(2).getBuffer(); + byte[] storedBytes3 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes3); + + storedBuffer = preppedPatterns.get(3).getBuffer(); + byte[] storedBytes4 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes4); + + assertEquals("QWord needs split to accommodate 16bit prog size", firstSplit, + HexFormat.of().formatHex(storedBytes1)); + assertEquals("QWord needs split to accommodate 16bit prog size", secondSplit, + HexFormat.of().formatHex(storedBytes2)); + assertEquals("QWord needs split to accommodate 16bit prog size", thirdSplit, + HexFormat.of().formatHex(storedBytes3)); + assertEquals("QWord needs split to accommodate 16bit prog size", fourthSplit, + HexFormat.of().formatHex(storedBytes4)); + } + + /** + * Simple test to ensure correct splitting of LE bytes when the DataType size is larger than the + * program size. + */ + @Test + public void testLittleEndianByteArraySingleSplit() { + + // assume bytes have already been parsed + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + // DataType=QWord, SearchType="Constant", parsed bytes + BpsSearchItem searchItem = + new BpsSearchItem("", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = leSearch(32, false); + + // prepare byte patterns based on search preferences indicated in the configObject + List preppedPatterns = patternTransformer.createPatterns(search, config); + + String firstSplit = "efcdab89"; + String secondSplit = "67452301"; + + BpsPattern firstSplitPattern = preppedPatterns.get(0); + ByteBuffer storedBuffer = firstSplitPattern.getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + BpsPattern secondSplitPattern = preppedPatterns.get(1); + storedBuffer = secondSplitPattern.getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + assertEquals("Pattern is split in half", 2, preppedPatterns.size()); + + // evaluate bytes are LE for each split + assertEquals("QWord needs reversed & split to accommodate 32bit prog size", firstSplit, + HexFormat.of().formatHex(storedBytes1)); + assertEquals("QWord needs reversed & to accommodate 32bit prog size", secondSplit, + HexFormat.of().formatHex(storedBytes2)); + + // evaluate description of the transformation + BpsTransformation firstSplitTransformations = firstSplitPattern.getTransformation(); + BpsTransformation secondSplitTransformations = secondSplitPattern.getTransformation(); + + assertEquals("This is the first split", 1, + firstSplitTransformations.splitPartIndex().intValue()); + assertEquals("Description needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 64] -> Pattern: EF CD AB 89 | Form: " + + "[Endian: Little -> Split to 32 Word Size | Chunk 1 of 2]", + firstSplitPattern.getDescription()); + + assertEquals("This is the second split", 2, + secondSplitTransformations.splitPartIndex().intValue()); + assertEquals("Description needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 64] -> Pattern: 67 45 23 01 | Form: " + + "[Endian: Little -> Split to 32 Word Size | Chunk 2 of 2]", + secondSplitPattern.getDescription()); + } + + /** + * Make sure LE splitting is happening generically, adjust search data type to DWord and program + * size to 16. + */ + @Test + public void testLittleEndianSplitDWord() { + + List bytePatterns = createPatterns("01234567", 32); + + BpsSearchItem searchItem = + new BpsSearchItem("Search 1", "key", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + BpsConfig config = leSearch(16, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String firstSplit = "6745"; + String secondSplit = "2301"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + assertEquals("LE DWord needs split to accommodate 16bit prog size", firstSplit, + HexFormat.of().formatHex(storedBytes1)); + + assertEquals("LE DWord needs split to accommodate 16bit prog size", secondSplit, + HexFormat.of().formatHex(storedBytes2)); + } + + /** + * Ensure correct splitting of LE bytes between byte arrays when the DataType size is larger + * than the program size -- in this case, multiple splits of the byte array will be needed. + * + */ + @Test + public void testLEMultiSplit() { + + // assume bytes have already been parsed, QWord + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = leSearch(16, false); // this will require 4 splits of the byte array + + // prepare byte string based on search preferences indicated in the configObject + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + assertEquals("There should be 4 splits, do not keep the original byte array", 4, + preppedPatterns.size()); + + String firstSplit = "efcd"; + String secondSplit = "ab89"; + String thirdSplit = "6745"; + String fourthSplit = "2301"; + + storedBuffer = preppedPatterns.get(0).getBuffer(); + storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + storedBuffer = preppedPatterns.get(2).getBuffer(); + byte[] storedBytes3 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes3); + + storedBuffer = preppedPatterns.get(3).getBuffer(); + byte[] storedBytes4 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes4); + + assertEquals("QWord needs LE & split to accommodate 16bit prog size", firstSplit, + HexFormat.of().formatHex(storedBytes1)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", secondSplit, + HexFormat.of().formatHex(storedBytes2)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", thirdSplit, + HexFormat.of().formatHex(storedBytes3)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", fourthSplit, + HexFormat.of().formatHex(storedBytes4)); + } + + /** + * In the case where the data type needs split because of the program size and we are searching + * both BE and LE - perform the little endian arrangement first and then split; keep BE + * arrangement. + */ + @Test + public void testLeAndBeNeedSplit() { + + // assume bytes have already been parsed + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + // DataType=QWord, SearchType="Constant", no additional tags from XML, parsed bytes + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = beLeSearch(32, false); + + // prepare byte string based on search preferences indicated in the configObject + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + // both BE and LE patterns are split into 2 + assertEquals("There should be 4 total byte arrays to search for", 4, + preppedPatterns.size()); + + String firstSplitLE = "efcdab89"; + String secondSplitLE = "67452301"; + + ByteBuffer storedBuffer = preppedPatterns.get(2).getBuffer(); + byte[] storedLeBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedLeBytes1); + + storedBuffer = preppedPatterns.get(3).getBuffer(); + byte[] storedBeBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBeBytes2); + + assertEquals("QWord needs reversed & split to accommodate 32bit prog size", firstSplitLE, + HexFormat.of().formatHex(storedLeBytes1)); + + assertEquals("QWord needs reversed & split to accommodate 32bit prog size", secondSplitLE, + HexFormat.of().formatHex(storedBeBytes2)); + } + + /** + * Test LE BE simultaneous multi split + */ + @Test + public void testBeLeMultiSplit() { + // assume bytes have already been parsed, QWord + String bytes = "0123456789abcdef"; + List bytePatterns = createPatterns(bytes, 64); + + BpsSearchItem searchItem = + new BpsSearchItem("Test", "key", 64, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + // these would be determined at runtime + BpsConfig config = beLeSearch(16, false); // this will require 4 splits of both byte arrays + + // prepare byte string based on search preferences indicated in the configObject + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals("There should be 8 splits, do not keep the original byte array", 8, + preppedPatterns.size()); + + String firstSplitBe = "0123"; + String secondSplitBe = "4567"; + String thirdSplitBe = "89ab"; + String fourthSplitBe = "cdef"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(0).getBuffer(); + storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + storedBuffer = preppedPatterns.get(2).getBuffer(); + byte[] storedBytes3 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes3); + + storedBuffer = preppedPatterns.get(3).getBuffer(); + byte[] storedBytes4 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes4); + + assertEquals("QWord needs split to accommodate 16bit prog size", firstSplitBe, + HexFormat.of().formatHex(storedBytes1)); + assertEquals("QWord needs split to accommodate 16bit prog size", secondSplitBe, + HexFormat.of().formatHex(storedBytes2)); + assertEquals("QWord needs split to accommodate 16bit prog size", thirdSplitBe, + HexFormat.of().formatHex(storedBytes3)); + assertEquals("QWord needs split to accommodate 16bit prog size", fourthSplitBe, + HexFormat.of().formatHex(storedBytes4)); + + String firstSplitLe = "efcd"; + String secondSplitLe = "ab89"; + String thirdSplitLe = "6745"; + String fourthSplitLe = "2301"; + + storedBuffer = preppedPatterns.get(4).getBuffer(); + storedBytes1 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes1); + + storedBuffer = preppedPatterns.get(5).getBuffer(); + storedBytes2 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes2); + + storedBuffer = preppedPatterns.get(6).getBuffer(); + storedBytes3 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes3); + + storedBuffer = preppedPatterns.get(7).getBuffer(); + storedBytes4 = new byte[storedBuffer.remaining()]; + storedBuffer.get(storedBytes4); + + assertEquals("QWord needs LE & split to accommodate 16bit prog size", firstSplitLe, + HexFormat.of().formatHex(storedBytes1)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", secondSplitLe, + HexFormat.of().formatHex(storedBytes2)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", thirdSplitLe, + HexFormat.of().formatHex(storedBytes3)); + assertEquals("QWord needs LE & split to accommodate 16bit prog size", fourthSplitLe, + HexFormat.of().formatHex(storedBytes4)); + + } + + /** + * Simple test to verify that the description string is correctly concatenated when multiple + * manipulations have been performed. + */ + @Test + public void testMultiManipulationAndMultiSplitDescriptionString() { + List bytePatterns = createPatterns("01234567", 32); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 32, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + BpsConfig config = leSearch(16, false); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + // evaluate descriptions at the pattern level and transformation level + BpsPattern firstSplitPattern = preppedPatterns.get(0); + assertEquals("Pattern description string needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 32] -> Pattern: 67 45 | Form: " + + "[Endian: Little -> Split to 16 Word Size | Chunk 1 of 2]", + firstSplitPattern.getDescription()); + BpsTransformation firstSplitTransformation = firstSplitPattern.getTransformation(); + assertEquals("Transformation Description string needs fixed", + "Endian: Little -> Split to 16 Word Size | Chunk 1 of 2", + firstSplitTransformation.getDescription()); + + BpsPattern secondSplitPattern = preppedPatterns.get(1); + assertEquals("Pattern description string needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 32] -> Pattern: 23 01 | Form: " + + "[Endian: Little -> Split to 16 Word Size | Chunk 2 of 2]", + secondSplitPattern.getDescription()); + BpsTransformation secondSplitTransformation = secondSplitPattern.getTransformation(); + assertEquals("Transformation Description string needs fixed", + "Endian: Little -> Split to 16 Word Size | Chunk 2 of 2", + secondSplitTransformation.getDescription()); + + } + + /** + * Simple test to zero-pad a 16bit word size to 32bit program size, BE. + */ + @Test + public void testSimpleBeBytePadding() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + BpsConfig config = beSearch(32, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String bytes = "00001122"; + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] paddedBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedBytes); + + assertEquals("BE padding is to the right of the byte", bytes, + HexFormat.of().formatHex(paddedBytes)); + + BpsTransformation patternTransformation = preppedPatterns.get(0).getTransformation(); + assertEquals("Transformation record must show padding size", + "Endian: Big -> Padded to 32", patternTransformation.getDescription()); + } + + /** + * Simple test to zero-pad a 16bit word size to 32bit program size, LE. + */ + @Test + public void testSimpleLeBytePadding() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + BpsConfig config = leSearch(32, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + String bytes = "22110000"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] paddedBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedBytes); + + assertEquals("LE padding is to the right of the byte", bytes, + HexFormat.of().formatHex(paddedBytes)); + + assertEquals("Transformation record must show LE arrangement and padding size", + "Endian: Little -> Padded to 32", + preppedPatterns.get(0).getTransformation().getDescription()); + } + + /** + * Zero-pad a 16bit word boundary using size quantization for 32 and 64bit word sizes to fit a + * 64bit program word boundary, BE. + */ + @Test + public void testPad16to64WordSize() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + BpsConfig config = beSearch(64, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + // when padding, we do not search using the original word boundary + assertEquals("2 new padded byte arrays should have been generated: 32 & 64", 2, + preppedPatterns.size()); + + String bytes32 = "00001122"; + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] padded32Bytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(padded32Bytes); + + BpsTransformation patternTransformation32 = preppedPatterns.get(0).getTransformation(); + assertEquals("32bit BE padding is to the left of the byte", bytes32, + HexFormat.of().formatHex(padded32Bytes)); + assertEquals("Transformation string must reflect pad 32 size", 32, + patternTransformation32.extensionSize().intValue()); + + String bytes64 = "0000000000001122"; + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] padded64Bytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(padded64Bytes); + + BpsTransformation patternTransformation64 = preppedPatterns.get(1).getTransformation(); + assertEquals("64bit BE padding is to the left of the byte", bytes64, + HexFormat.of().formatHex(padded64Bytes)); + assertEquals("Transformation string must reflect pad 64 size", 64, + patternTransformation64.extensionSize().intValue()); + } + + /** + * Zero-pad a 16bit word boundary using size quantization for 32 and 64bit word sizes to fit a + * 64bit program word boundary, LE. + */ + @Test + public void testPad16to64WordSizeLE() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + + BpsConfig config = leSearch(64, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals( + "2 byte arrays should have been generated: LE-32pad, LE-64pad", 2, + preppedPatterns.size()); + + String bytes32 = "22110000"; + BpsPattern pattern32 = preppedPatterns.get(0); + ByteBuffer storedBuffer = pattern32.getBuffer(); + byte[] padded32Bytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(padded32Bytes); + + BpsTransformation patternTransformation32 = pattern32.getTransformation(); + assertEquals("32bit LE padding is to the left of the byte", bytes32, + HexFormat.of().formatHex(padded32Bytes)); + assertEquals("Transformation string must reflect pad 32 size", 32, + patternTransformation32.extensionSize().intValue()); + assertEquals("Transformation string must reflect LE", Endian.LITTLE, + patternTransformation32.endianness()); + + String bytes64 = "2211000000000000"; + BpsPattern pattern64 = preppedPatterns.get(1); + storedBuffer = pattern64.getBuffer(); + byte[] padded64Bytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(padded64Bytes); + BpsTransformation patternTransformation64 = pattern64.getTransformation(); + assertEquals("64bit LE padding is to the left of the byte", bytes64, + HexFormat.of().formatHex(padded64Bytes)); + assertEquals("Transformation string must reflect pad 64 size", 64, + patternTransformation64.extensionSize().intValue()); + assertEquals("Transformation string must reflect LE", Endian.LITTLE, + patternTransformation64.endianness()); + } + + /** + * If the user wishes to search both BE and LE and also wants to extend the data types, need to + * make sure that 2 new rearranged arrays are generated - both zero-padded, one BE one LE. + */ + @Test + public void testPadBEandLE() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + BpsConfig config = beLeSearch(32, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals("2 byte arrays should have been generated", 2, preppedPatterns.size()); + + String bytesBeExtended = "00001122"; + String bytesLeExtended = "22110000"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] paddedBEBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedBEBytes); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] paddedLEBytes = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedLEBytes); + + assertEquals("LE padding is to the right of the byte", bytesLeExtended, + HexFormat.of().formatHex(paddedLEBytes)); + assertEquals("BE padding is to the left of the byte", bytesBeExtended, + HexFormat.of().formatHex(paddedBEBytes)); + } + + /** + * Pad BE and LE patterns from 16 to 64 word boundary: generate BE and LE patterns with 32 and + * 64 word boundaries. + */ + @Test + public void testPadBEandLEQuantization() { + List bytePatterns = createPatterns("1122", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + + List search = makeSearchCollection(searchItem); + BpsConfig config = beLeSearch(64, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + assertEquals("4 byte arrays should have been generated", 4, preppedPatterns.size()); + + String bytesBeExtended32 = "00001122"; + String bytesBeExtended64 = "0000000000001122"; + String bytesLeExtended32 = "22110000"; + String bytesLeExtended64 = "2211000000000000"; + + ByteBuffer storedBuffer = preppedPatterns.get(0).getBuffer(); + byte[] paddedBEBytes32 = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedBEBytes32); + + storedBuffer = preppedPatterns.get(1).getBuffer(); + byte[] paddedBEBytes64 = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedBEBytes64); + + storedBuffer = preppedPatterns.get(2).getBuffer(); + byte[] paddedLEBytes32 = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedLEBytes32); + + storedBuffer = preppedPatterns.get(3).getBuffer(); + byte[] paddedLEBytes64 = new byte[storedBuffer.remaining()]; + storedBuffer.get(paddedLEBytes64); + + assertEquals("LE padding is to the right of the byte", bytesLeExtended32, + HexFormat.of().formatHex(paddedLEBytes32)); + assertEquals("BE padding is to the left of the byte", bytesBeExtended32, + HexFormat.of().formatHex(paddedBEBytes32)); + assertEquals("LE padding is to the right of the byte", bytesLeExtended64, + HexFormat.of().formatHex(paddedLEBytes64)); + assertEquals("BE padding is to the left of the byte", bytesBeExtended64, + HexFormat.of().formatHex(paddedBEBytes64)); + } + + @Test + public void testPaddingQuantizationDescriptionString() { + List bytePatterns = createPatterns("01234567", 16); + BpsSearchItem searchItem = + new BpsSearchItem("Test pattern", "key", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem.setBytePatterns(bytePatterns); + List search = makeSearchCollection(searchItem); + + BpsConfig config = beSearch(64, true); + + List preppedPatterns = + patternTransformer.createPatterns(search, config); + + BpsPattern pattern32 = preppedPatterns.get(0); + assertEquals("Transformation string needs fixed", + "Endian: Big -> Padded to 32", pattern32.getTransformation().getDescription()); + + assertEquals("Pattern description string needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 16] -> Pattern: 01 23 45 67 | Form: " + + "[Endian: Big -> Padded to 32]", + pattern32.getDescription()); + + BpsPattern pattern64 = preppedPatterns.get(1); + assertEquals("Transformation string needs fixed", + "Endian: Big -> Padded to 64", pattern64.getTransformation().getDescription()); + + assertEquals("Pattern description string needs fixed", + "[Pattern Name: ParentSearchItem | Word Size: 16] -> Pattern: 00 00 00 00 01 23 45 67 | Form: " + + "[Endian: Big -> Padded to 64]", + pattern64.getDescription()); + } + + /** + * Test the data flow from the parser through pattern preparation. + * + * @throws Exception xml parse exception + */ + @Test + public void testParserToLEManipulation() throws Exception { + String xmlFile = + "ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml"; + File testXmlFile = ResourceManager.getResourceFile(xmlFile); + List searchCollection = + BpsXmlParser.parseSearchFile(testXmlFile, TaskMonitor.DUMMY); + + assertEquals("Should be 9 family collections of SearchItems", 9, searchCollection.size()); + assertEquals("Original patterns not parsed correctly", 85, + getPatternCount(searchCollection)); + + // these would be determined at runtime + BpsConfig config = beLeSearch(64, false); + + List preparedPatterns = + patternTransformer.createPatterns(searchCollection, config); + + /* + * Pattern count break down: + * 85 total patterns parsed x 2 (BE & LE) = 170 + * 36 patterns are 8 bit (single byte) and will not need transformed to LE: 170-36 = 134 + * 85 total patterns parsed x 2 (made into LE) - 36 byte word size = 134 + */ + assertEquals("Additional patterns added from LE manipulation", 134, + preparedPatterns.size()); + } + + /** + * Helper to count the total number of patterns across all searchItems within a search + * collection. + * + * @param searchCollection list of searches + * + * @return total pattern count + */ + private int getPatternCount(List searchCollection) { + + int totalPatternCount = 0; + for (BpsSearch searchItems : searchCollection) { + for (BpsSearchItem item : searchItems.getSearchItems()) { + totalPatternCount += item.getPatterns().size(); + } + } + return totalPatternCount; + } + + /** + * Test pattern file with word size byte. Edge case. + * + * @throws Exception xml parse exception + */ + @Test + public void testHandleWordSizeByteProperly() throws Exception { + String xmlFile = + "ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml"; + File testXmlFile = ResourceManager.getResourceFile(xmlFile); + List searchCollection = + BpsXmlParser.parseSearchFile(testXmlFile, TaskMonitor.DUMMY); + + assertEquals("Should be 2 family collections of SearchItems", 2, searchCollection.size()); + assertEquals("Original patterns not parsed correctly", 15, + getPatternCount(searchCollection)); + + // these would be determined at runtime + BpsConfig config = beLeSearch(64, false); + + List preppedPatterns = + patternTransformer.createPatterns(searchCollection, config); + + /* + * One pattern contains word size of byte (8bit) will not reverse to LE + * Original pattern count: 15 + * Patterns made LE: 14 (1 pattern will not need reversed) + * Total patterns for search: 15+14 = 29 + */ + assertEquals("Additional patterns added from LE manipulation", 29, preppedPatterns.size()); + } + + /** + * {@link BpsSearch} items should be skippable using the generated key at the + * {@link BpsSearchItem} level. + */ + @Test + public void testSkipSearchItem() { + byte[] rawBytes = HexFormat.of().parseHex("0101"); + ByteBuffer buffer = ByteBuffer.wrap(rawBytes); + + byte[] rawBytes2 = HexFormat.of().parseHex("ABCD"); + ByteBuffer buff2 = ByteBuffer.wrap(rawBytes2); + + BpsSearchItem searchItem = + new BpsSearchItem("SearchItem1", "SearchItemkey0", 16, + BpsSearchType.CONSTANT_SEARCH); + BpsPattern pattern = + new BpsPattern("Bytes", buffer, searchItem, BpsTransformation.empty()); + + List bytePatterns = new ArrayList<>(); + bytePatterns.add(pattern); + searchItem.setBytePatterns(bytePatterns); + + BpsSearchItem searchItem2 = + new BpsSearchItem("SearchItem2", "SearchItemkey1", 16, BpsSearchType.CONSTANT_SEARCH); + BpsPattern pattern2 = + new BpsPattern("Bytes", buff2, searchItem2, BpsTransformation.empty()); + + List bytePatterns2 = new ArrayList<>(); + bytePatterns2.add(pattern2); + searchItem2.setBytePatterns(bytePatterns2); + + List searchItems = new ArrayList<>(); + searchItems.add(searchItem); + searchItems.add(searchItem2); + + BpsSearch search = new BpsSearch("Search1", "Searchkey1", searchItems); + + List searchList = new ArrayList<>(); + searchList.add(search); + + BpsConfig config = beSearch(32, false); + + Set skipList = new HashSet<>(); + skipList.add("SearchItemkey0"); + + List preppedPatterns = + patternTransformer.createPatterns(searchList, skipList, config); + assertEquals("1 pattern should have been skipped.", 1, preppedPatterns.size()); + assertFalse(preppedPatterns.contains(searchItem.getPatterns().get(0))); + assertTrue(preppedPatterns.contains(searchItem2.getPatterns().get(0))); + } + + /** + * Whole {@link BpsSearch} are also skippable + */ + @Test + public void testSkipSearch() { + byte[] rawBytes = HexFormat.of().parseHex("0101"); + ByteBuffer buffer = ByteBuffer.wrap(rawBytes); + + BpsSearchItem searchItem = + new BpsSearchItem("SearchItem1", "SearchItemkey0", 16, + BpsSearchType.CONSTANT_SEARCH); + BpsPattern pattern = + new BpsPattern("Bytes", buffer, searchItem, BpsTransformation.empty()); + + List bytePatterns = new ArrayList<>(); + bytePatterns.add(pattern); + searchItem.setBytePatterns(bytePatterns); + + BpsSearchItem searchItem2 = + new BpsSearchItem("SearchItem2", "SearchItemkey1", 16, BpsSearchType.CONSTANT_SEARCH); + searchItem2.setBytePatterns(bytePatterns); + + List searchItems = new ArrayList<>(); + searchItems.add(searchItem); + searchItems.add(searchItem2); + + BpsSearch search = new BpsSearch("Search1", "Searchkey1", searchItems); + + List searchList = new ArrayList<>(); + searchList.add(search); + + BpsConfig config = beSearch(32, false); + + Set skipList = new HashSet<>(); + skipList.add("Searchkey1"); + + List preppedPatterns = + patternTransformer.createPatterns(searchList, skipList, config); + assertEquals("All patterns should have been skipped.", 0, preppedPatterns.size()); + } + + /** + * Ensure generated keys can be used for skipping multiple things (both at the Search and + * SearchItem level). + * + * @throws SAXException for XML parsing + */ + @Test + public void testMultiSkipFromGeneratedKey() throws SAXException { + String xmlFile = + "ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml"; + File testXmlFile = ResourceManager.getResourceFile(xmlFile); + List searchCollection = + BpsXmlParser.parseSearchFile(testXmlFile, TaskMonitor.DUMMY); + + assertEquals("Full pattern count is wrong", 85, getPatternCount(searchCollection)); + + // these would be determined at runtime + BpsConfig config = beSearch(64, false); + + // the GUI will build and maintain this list from the user's interactions + Set skipList = new HashSet<>(); + skipList.add(searchCollection.get(0).getId()); // skip the first Search all together + + /* + * Skip the 2nd SearchItem in the 2nd Search - this contains 4 patterns, starting on line 14 + * + 67e6096a85ae67bb + 67e6096a + 85ae67bb + 67e6096a85ae67bb + + */ + skipList.add(searchCollection.get(1).getSearchItems().get(1).getId()); + + List preparedPatterns = + patternTransformer.createPatterns(searchCollection, skipList, config); + + /* + * We are skipping: + * - first search (which contains 1 search item): 1 pattern + * - second searchItem from the second search: 4 patterns + * Total skipped patterns = 5 + * Total search patterns: 85-5 = 80 + */ + assertEquals( + "First search and 2nd search item from 2nd search should have been skipped - 5 less " + + "patterns.", + 80, preparedPatterns.size()); + } +} diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsParserTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsParserTest.java new file mode 100644 index 0000000000..9039b54fd2 --- /dev/null +++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/bytepatternsearch/BpsParserTest.java @@ -0,0 +1,260 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ghidra.util.bytesearch.bytepatternsearch; + +import static org.junit.Assert.*; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.HexFormat; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; + +import generic.test.AbstractGenericTest; +import ghidra.util.task.TaskMonitor; +import resources.ResourceManager; + +public class BpsParserTest extends AbstractGenericTest { + + private String xmlFile = "ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml"; + private File testXmlFile; + private List searchCollection; + + @Before + public void setUp() throws Exception { + testXmlFile = ResourceManager.getResourceFile(xmlFile); + searchCollection = BpsXmlParser.parseSearchFile(testXmlFile, TaskMonitor.DUMMY); + } + + /** + * Simple test for successful loading and parsing of example XML. + */ + @Test + public void testParserLoadXml() { + + BpsSearch search1 = searchCollection.get(0); + assertEquals("SearchID wasn't generated correctly", "basic_search_collection.xml_Search1_1", + search1.getId()); + assertEquals("1", search1.getDescription()); + assertEquals("Search1", search1.getSearchName()); + assertEquals("test", search1.getSubmitter()); + assertEquals("Search1", search1.toString()); + assertEquals("Incorrect number of search items parsed for collection", 1, + search1.getSearchItems().size()); + + BpsSearchItem searchItem = search1.getSearchItems().get(0); + assertEquals("SearchIteam Id wasn't generated correctly", + "basic_search_collection.xml_Constant 1_0", searchItem.getId()); + assertEquals("Human", searchItem.getSubmitter()); + assertEquals(searchItem.getSearchType(), BpsSearchType.CONSTANT_SEARCH); + assertEquals("Simple constant search", searchItem.getDescription()); + assertEquals("Word size wasn't parsed correctly", 64, searchItem.getWordSize()); // QWord becomes 64 + } + + /** + * Ensure that multiple collections with multiple searchItems are being parsed correctly + */ + @Test + public void testMultiCollectionParse() { + assertEquals("Incorrect number of search collections parsed", 9, + searchCollection.size()); + + /* + Search Collection 3 + + + ... + + + // 8 Search items + + */ + + BpsSearch search3 = searchCollection.get(2); + assertEquals("Collection 3 should have 8 search items", 8, search3.getSearchItems().size()); + + /* + Search Collection 5 + + + + 67 + ... + ... + // 34 Bytes elements + */ + + BpsSearch search5 = searchCollection.get(4); + List searchItems = search5.getSearchItems(); + BpsSearchItem searchItem = searchItems.get(0); + List bytes = searchItem.getPatterns(); + assertEquals("Collection 5 should have 1 search item with 34 byte tags", 34, bytes.size()); + } + + /* + * The current legacy object, and XML, follows a construct with multiple tag types. We need to + * respect these tag types and store them as an attribute under "TagType" so that they can be + * easily retrieved to handle the bytes appropriately. Some tags are expected to occur + * in tandem with each other in the same SearchItem. + *

    + * NOTE: future design is to eliminate multiple tag types and assign attributes to byte tags as + * the previous test assumes for. + */ + @Test + public void testByteTagsConvertedToAttributesToFitLegacyObject() { + + int fluxCapacitor = 7; + BpsSearch search = searchCollection.get(fluxCapacitor); + List searchItems = search.getSearchItems(); + BpsSearchItem searchItem = searchItems.get(0); + BpsPattern bytes = searchItem.getPatterns().get(0); + String tagName = bytes.getName(); + + assertEquals("Parser handles tags around byte strings as attributes", + "Bytes", tagName); + } + + /** + * When each byte string is parsed, it should be stored according to the indicated DataType + * attribute on the {@code} tag. + */ + @Test + public void testVerifyBytesParsedAccordingToXmlDataType() { + String originalHexString = "67e6096a85ae67bb"; + + BpsSearchItem searchItem = searchCollection.get(0).getSearchItems().get(0); + BpsPattern bytes = searchItem.getPatterns().get(0); + + // QWord + ByteBuffer byteBuffer = bytes.getBuffer(); + byte[] byteArray = new byte[byteBuffer.remaining()]; + byteBuffer.get(byteArray); + assertEquals("Parsed bytes do not match original set", originalHexString, + HexFormat.of().formatHex(byteArray)); + + // Word + int fluxCapacitor = 5; + searchItem = searchCollection.get(fluxCapacitor).getSearchItems().get(0); + bytes = searchItem.getPatterns().get(0); + byteBuffer = bytes.getBuffer(); + byteArray = new byte[byteBuffer.remaining()]; + byteBuffer.get(byteArray); + assertEquals("Parsed bytes do not match original set", "67e6", + HexFormat.of().formatHex(byteArray)); + } + + /** + * For ease of tracking search item details later, keep a link between the byte pattern and its + * parent search item. + */ + @Test + public void testPatternLinkBacktoParentSearchItem() { + BpsSearchItem parentItem = + searchCollection.get(0).getSearchItems().get(0); + BpsPattern childPattern = parentItem.getPatterns().get(0); + + assertEquals("Patterns should link to their parent search item.", parentItem, + childPattern.getSearchItem()); + assertEquals("Parent item knows the search type", + childPattern.getSearchItem().getSearchType(), BpsSearchType.CONSTANT_SEARCH); + } + + /** + * Generate search type objects for tracking search-specific information and use later as + * part of search post-processing. + */ + @Test + public void testSearchTypeObjectGeneration() { + BpsSearchItem searchItem = searchCollection.get(0).getSearchItems().get(0); + assertEquals("Wrong (or no) search type object created", searchItem.getSearchType(), + BpsSearchType.CONSTANT_SEARCH); + + searchItem = searchCollection.get(1).getSearchItems().get(0); + assertEquals("Wrong (or no) search type object created", searchItem.getSearchType(), + BpsSearchType.TABLE_SEARCH); + } + + @Test + public void testAdditionalAttributesOnSearchandSearchItems() { + int searchIndex = 2; + BpsSearch search = searchCollection.get(searchIndex); + assertEquals("Search3 tag has 2 additional attributes", 2, + search.getAdditionalAttributes().size()); + + BpsSearchItem searchItem = search.getSearchItems().get(0); + assertEquals("SearchItem has 1 additional attribute (Reference)", 1, + searchItem.getAdditionalAttributes().size()); + } + + @Test + public void testParsingErrorHandler() throws SAXException { + String currXmlFile = "ghidra/util/bytesearch/bytepatternsearch/parse_errors.xml"; + File currTestXmlFile = ResourceManager.getResourceFile(currXmlFile); + String message = ""; + int errorCount = 0; + List errors = null; + try { + searchCollection = BpsXmlParser.parseSearchFile(currTestXmlFile, TaskMonitor.DUMMY); + fail("Should have thrown a SAXException but did not"); + } + catch (BpsXmlValidationException e) { + message = e.getMessage(); + errorCount = e.getErrorCount(); + errors = e.getErrors(); + } + assertEquals(12, errorCount); + // error with wrapper tag + assertTrue(message.contains("XML parsing failed")); + assertTrue( + message.contains( + "Pattern file must have a wrapper tag to start.")); + // error with level tag + assertTrue(message.contains(" tag expected")); + // ensure correct errors were collected + assertEquals( + "org.xml.sax.SAXParseException; Line: 54, Col: 11: Byte string is not an even size and cannot be parsed as Hex. Either adjust WordSize attribute to ASCII or zero-pad hex to make it well-formed. Skipping entire search item.", + errors.get(8).toString()); + } + + /** + * Verify successful loading of provided sample file with slightly different schema. + * + * @throws SAXException for parsing exception + */ + @Test + public void testProvidedSamplePatternFile() throws SAXException { + String currXmlFile = + "ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml"; + File currTestXmlFile = ResourceManager.getResourceFile(currXmlFile); + try { + searchCollection = BpsXmlParser.parseSearchFile(currTestXmlFile, TaskMonitor.DUMMY); + } + catch (BpsXmlValidationException e) { + fail("This file should parse without errors to prove compatibility"); + } + BpsSearchItem item = searchCollection.get(0).getSearchItems().get(0); + + assertEquals("SearchItem was not parsed correctly.", "DIP32", + item.getPatternName()); + assertEquals("Search tag name type isn't correct", "FunctionConstant", + item.getPatterns().get(0).getName()); + } +} diff --git a/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml new file mode 100644 index 0000000000..4bc66ba714 --- /dev/null +++ b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/basic_search_collection.xml @@ -0,0 +1,153 @@ + + + + 67e6096a85ae67bb + + + + + 67e6096a + 85ae67bb + 67e6096a + 85ae67bb + + + 67e6096a85ae67bb + 67e6096a + 85ae67bb + 67e6096a85ae67bb + + + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + 67e6096a85ae67bb + 67e6096a85ae67bb + + + + + 67 + e6 + 09 + 6a + 85 + ae + 67 + bb + 67 + e6 + 09 + 6a + 85 + ae + 67 + bb + 67 + e6 + 09 + 6a + 85 + ae + 67 + bb + 67 + e6 + 09 + 6a + 85 + ae + 67 + bb + 67 + e6 + + + + + 67e6 + 096a + 85ae + 67bb + + + + + 67e6096a85ae67bb + + + + + 01 23 + 45 67 + 89 ab + + + + + hello + world + + + \ No newline at end of file diff --git a/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml new file mode 100644 index 0000000000..a781d0fa98 --- /dev/null +++ b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/dword_qword_for32bit_pattern.xml @@ -0,0 +1,35 @@ + + + + f0f0f0f0 + fff0000f + 33333333 + 03fc03fc + aaaaaaaa + + + f0f0f0f0fff0000f + 3333333303fc03fc + aaaaaaaafcfcfcfc + + + + 00 00 01 01 01 01 01 01 00 01 01 01 01 01 01 00 + + + + 02080800 00080000 02000002 02080802 02000000 00080802 00080002 02000002 00080802 02080800 02080000 00000802 02000802 02000000 00000000 00080002 00080000 00000002 02000800 00080800 02080802 02080000 00000802 02000800 00000002 00000800 00080800 02080002 00000800 02000802 02080002 00000000 00000000 02080802 02000800 00080002 02080800 00080000 00000802 02000800 02080002 00000800 00080800 02000002 00080802 00000002 02000002 02080000 02080802 00080800 02080000 02000802 02000000 00000802 00080002 00000000 00080000 02000000 02000802 02080800 00000002 02080002 00000800 00080802 + + + + + 00002b992ddfa232 + ffffffffffffffff + 00002b992ddfa233 + + + FFFFD466D2205DCD + 00002B992DDFA232 + + + diff --git a/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/parse_errors.xml b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/parse_errors.xml new file mode 100644 index 0000000000..bac238ea1b --- /dev/null +++ b/Ghidra/Features/Base/src/test/resources/ghidra/util/bytesearch/bytepatternsearch/parse_errors.xml @@ -0,0 +1,101 @@ + + + + 00 00 + + + + + 00 00 + + + + + 00 00 + + + + + 00 00 + + + + + 00 00 00 00 00 00 + + + + + 00 00 00 00 00 00 + + + 00 00 00 00 00 00 + + + + + 00 00 00 00 00 00 + + + 00 00 00 00 00 00 + + + + + 00 00 00 00 00 00 + + + 00 00 00 00 00 00 + + + + + 67e6096 + 67e6096a + + + + + 67e60960 + 67e6096a + + + + + 67e6096a + 67e6096a + + + 67e60a96 + 67e6096a + + + + + 67e6096a + 67e6096a + + + 67e60a96 + 67e6096a + + + + + 67e6096a + 67e6096a + + + + + 67e6096a + 67e6096a + + + + + 67e6096a + 67e6096a + + + \ No newline at end of file