From 39982f9984c0004c17bce327d88c1d7e1d8e7f36 Mon Sep 17 00:00:00 2001 From: James <49045138+ghidracadabra@users.noreply.github.com> Date: Fri, 8 Dec 2023 12:43:01 -0500 Subject: [PATCH] GP-4110 cleaning up bsim scripts --- .../CompareBSimSignaturesScript.java | 174 ++++++++++++++++++ ...areBSimSignaturesSpecifyWeightsScript.java | 63 +++++++ ...les.java => CompareExecutablesScript.java} | 9 +- .../ghidra_scripts/CompareSignatures.java | 148 --------------- .../CompareSignaturesSpecifyWeights.java | 155 ---------------- .../BSim/ghidra_scripts/DebugSignatures.java | 72 -------- .../DumpBSimDebugSignaturesScript.java | 37 ++++ .../DumpBSimDebugSignaturesScript.py | 48 +++++ .../DumpBSimSignaturesScript.java | 80 ++++++++ ...natures.py => DumpBSimSignaturesScript.py} | 42 +++-- .../BSim/ghidra_scripts/DumpSignatures.java | 115 ------------ .../BSim/ghidra_scripts/DumpSignatures.py | 61 ------ ...y.java => ExampleOverviewQueryScript.java} | 17 +- ...Query.py => ExampleOverviewQueryScript.py} | 6 +- ...ent.java => ExampleQueryClientScript.java} | 48 ++++- .../ghidra_scripts/GenerateSignatures.java | 43 +++-- .../BSim/ghidra_scripts/GenerateSignatures.py | 4 +- .../BSim/ghidra_scripts/QueryFunction.java | 19 +- .../BSim/ghidra_scripts/QueryFunction.py | 6 +- .../QueryWithFiltersScript.java | 20 +- .../ghidra_scripts/QueryWithFiltersScript.py | 173 ----------------- .../BSim/ghidra_scripts/TailoredAnalysis.java | 15 +- 22 files changed, 538 insertions(+), 817 deletions(-) create mode 100755 Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesScript.java create mode 100755 Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesSpecifyWeightsScript.java rename Ghidra/Features/BSim/ghidra_scripts/{CompareExecutables.java => CompareExecutablesScript.java} (94%) delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/CompareSignatures.java delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/CompareSignaturesSpecifyWeights.java delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/DebugSignatures.java create mode 100755 Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.java create mode 100755 Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.py create mode 100755 Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.java rename Ghidra/Features/BSim/ghidra_scripts/{DumpDebugSignatures.py => DumpBSimSignaturesScript.py} (60%) delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.java delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.py rename Ghidra/Features/BSim/ghidra_scripts/{ExampleOverviewQuery.java => ExampleOverviewQueryScript.java} (89%) rename Ghidra/Features/BSim/ghidra_scripts/{ExampleOverviewQuery.py => ExampleOverviewQueryScript.py} (89%) rename Ghidra/Features/BSim/ghidra_scripts/{ExampleQueryClient.java => ExampleQueryClientScript.java} (61%) delete mode 100755 Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.py diff --git a/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesScript.java b/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesScript.java new file mode 100755 index 0000000000..baa95b722b --- /dev/null +++ b/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesScript.java @@ -0,0 +1,174 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This script will print information about a BSim comparison of two functions. Navigate to the +// first function, run this script, navigate to the second function, and run the script again. +// The two functions can be in different (open) programs. Subsequent runs of the script will +// compare the current function and the previous function. For each comparison, the BSim signature +// weights file is chosen based on the architecture of the current function. +//@category BSim + +import java.io.*; + +import org.xml.sax.SAXException; + +import generic.jar.ResourceFile; +import generic.lsh.vector.*; +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileOptions; +import ghidra.app.decompiler.signature.SignatureResult; +import ghidra.app.script.GhidraScript; +import ghidra.app.services.ProgramManager; +import ghidra.features.bsim.query.GenSignatures; +import ghidra.program.model.address.Address; +import ghidra.program.model.lang.LanguageID; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.Program; +import ghidra.util.xml.SpecXmlUtils; +import ghidra.xml.NonThreadedXmlPullParserImpl; +import ghidra.xml.XmlPullParser; +import utilities.util.reflection.ReflectionUtilities; + +public class CompareBSimSignaturesScript extends GhidraScript { + + protected LSHVectorFactory vectorFactory; + + @Override + protected void run() throws Exception { + if (isRunningHeadless()) { + popup("This script must be run from the Ghidra GUI"); + return; + } + if (currentProgram == null) { + popup("This script requires an open program"); + return; + } + Function func = getFunctionContaining(currentAddress); + if (func == null) { + popup("currentAddress must be in a function for this script."); + return; + } + ProgramManager programManager = state.getTool().getService(ProgramManager.class); + String lastProgramIdString = System.getProperty("ghidra.lastprogram"); + if (lastProgramIdString == null) { + setProperties(func); + return; + } + Program lastProgram = + getProgram(programManager.getAllOpenPrograms(), Long.parseLong(lastProgramIdString)); + if (lastProgram == null) { + setProperties(func); + return; + } + String addrstring = System.getProperty("ghidra.lastaddress"); + if (addrstring == null) { + setProperties(func); + return; + } + Address addr = lastProgram.getAddressFactory().getAddress(addrstring); + Function lastfunction = lastProgram.getFunctionManager().getFunctionAt(addr); + if (lastfunction == null) { + setProperties(func); + return; + } + if (!buildLSHVectorFactory()) { + return; + } + LSHVector vec = generateVector(func, currentProgram); + LSHVector lastvector = generateVector(lastfunction, lastProgram); + VectorCompare veccompare = new VectorCompare(); + double sim = lastvector.compare(vec, veccompare); + double signif = vectorFactory.calculateSignificance(veccompare); + StringBuilder buf = new StringBuilder(); + buf.append("Comparison results:\n"); + buf.append(lastProgram.getName()); + buf.append("."); + buf.append(lastfunction.getName()); + buf.append(" vs. "); + buf.append(currentProgram.getName()); + buf.append("."); + buf.append(func.getName()); + buf.append("\n Similarity: "); + buf.append(Double.toString(sim)); + buf.append("\n Significance: "); + buf.append(Double.toString(signif)); + buf.append("\n"); + lastvector.compareDetail(vec, buf); + println(buf.toString()); + } + + private void setProperties(Function func) { + System.setProperty("ghidra.lastprogram", + Long.toString(currentProgram.getUniqueProgramID())); + String addrstring = func.getEntryPoint().toString(); + System.setProperty("ghidra.lastaddress", addrstring); + } + + private LSHVector generateVector(Function f, Program program) { + DecompInterface decompiler = new DecompInterface(); + try { + decompiler.setOptions(new DecompileOptions()); + decompiler.toggleSyntaxTree(false); + decompiler.setSignatureSettings(vectorFactory.getSettings()); + if (!decompiler.openProgram(program)) { + println("Unable to initalize the Decompiler interface"); + println(decompiler.getLastMessage()); + return null; + } + SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null); + LSHVector vec = vectorFactory.buildVector(sigres.features); + return vec; + } + finally { + decompiler.closeProgram(); + decompiler.dispose(); + } + } + + private Program getProgram(Program[] progarray, long id) { + if (progarray == null) { + return null; + } + for (Program prog : progarray) { + if (prog.getUniqueProgramID() == id) { + return prog; + } + } + return null; + } + + protected static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile) + throws FileNotFoundException, IOException, SAXException { + InputStream input = weightsFile.getInputStream(); + XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser", + SpecXmlUtils.getXmlHandler(), false); + vectorFactory.readWeights(parser); + input.close(); + } + + protected boolean buildLSHVectorFactory() { + vectorFactory = new WeightedLSHCosineVectorFactory(); + try { + LanguageID id = currentProgram.getLanguageID(); + ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id); + readWeights(vectorFactory, defaultWeightsFile); + } + catch (IOException | SAXException e) { + printerr("Unexpected Exception..."); + printerr(ReflectionUtilities.stackTraceToString(e)); + } + return true; + } +} diff --git a/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesSpecifyWeightsScript.java b/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesSpecifyWeightsScript.java new file mode 100755 index 0000000000..4e42ab7a8f --- /dev/null +++ b/Ghidra/Features/BSim/ghidra_scripts/CompareBSimSignaturesSpecifyWeightsScript.java @@ -0,0 +1,63 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This script will print information about a BSim comparison of two functions. Navigate to the +// first function, run this script, navigate to the second function, and run the script again. +// The two functions can be in different (open) programs. Subsequent runs of the script will +// compare the current function and the previous function. For each comparison, the user selects +// which weights file to use. +//@category BSim + +import java.io.IOException; + +import org.xml.sax.SAXException; + +import generic.jar.ResourceFile; +import generic.lsh.vector.WeightedLSHCosineVectorFactory; +import ghidra.features.base.values.GhidraValuesMap; +import ghidra.framework.Application; +import ghidra.util.exception.CancelledException; +import utilities.util.reflection.ReflectionUtilities; + +public class CompareBSimSignaturesSpecifyWeightsScript extends CompareBSimSignaturesScript { + + private static final String[] WEIGHT_FILES = { "lshweights_nosize.xml", "lshweights_32.xml", + "lshweights_64.xml", "lshweights_cpool.xml" }; + + @Override + protected boolean buildLSHVectorFactory() { + vectorFactory = new WeightedLSHCosineVectorFactory(); + GhidraValuesMap values = new GhidraValuesMap(); + values.defineChoice("weights file", WEIGHT_FILES[0], WEIGHT_FILES); + try { + askValues("Select Weights File", "Select Weights File", values); + } + catch (CancelledException e) { + return false; + } + String weightsFile = values.getChoice("weights file"); + ResourceFile defaultWeightsFile = Application.findDataFileInAnyModule(weightsFile); + try { + readWeights(vectorFactory, defaultWeightsFile); + } + catch (IOException | SAXException e) { + printerr("Unexpected Exception..."); + printerr(ReflectionUtilities.stackTraceToString(e)); + return false; + } + return true; + + } +} diff --git a/Ghidra/Features/BSim/ghidra_scripts/CompareExecutables.java b/Ghidra/Features/BSim/ghidra_scripts/CompareExecutablesScript.java similarity index 94% rename from Ghidra/Features/BSim/ghidra_scripts/CompareExecutables.java rename to Ghidra/Features/BSim/ghidra_scripts/CompareExecutablesScript.java index cd88d84b0e..1b01cf79b9 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/CompareExecutables.java +++ b/Ghidra/Features/BSim/ghidra_scripts/CompareExecutablesScript.java @@ -25,19 +25,18 @@ import ghidra.features.bsim.query.FunctionDatabase; import ghidra.features.bsim.query.client.*; import ghidra.features.bsim.query.description.ExecutableRecord; -public class CompareExecutables extends GhidraScript { +public class CompareExecutablesScript extends GhidraScript { private ExecutableComparison exeCompare; + @Override protected void run() throws Exception { URL url = BSimClientFactory.deriveBSimURL("ghidra://localhost/repo"); try (FunctionDatabase database = BSimClientFactory.buildClient(url, true)) { // FileScoreCaching cache = new FileScoreCaching("/tmp/test_scorecacher.txt"); TableScoreCaching cache = new TableScoreCaching(database); - exeCompare = - new ExecutableComparison(database, 1000000, "11111111111111111111111111111111", - cache, - monitor); + exeCompare = new ExecutableComparison(database, 1000000, + "11111111111111111111111111111111", cache, monitor); // Specify the list of executables to compare by giving their md5 hash // exeCompare.addExecutable("22222222222222222222222222222222"); // 32 hex-digit string // exeCompare.addExecutable("33333333333333333333333333333333"); diff --git a/Ghidra/Features/BSim/ghidra_scripts/CompareSignatures.java b/Ghidra/Features/BSim/ghidra_scripts/CompareSignatures.java deleted file mode 100755 index 21dc52125d..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/CompareSignatures.java +++ /dev/null @@ -1,148 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// Use the decompiler to generate a signature for the current function containing the cursor -// If we remember the last signature that was generated, compare this signature with -// the last signature and print the similarity -//@category BSim - -import java.io.*; - -import org.xml.sax.SAXException; - -import generic.jar.ResourceFile; -import generic.lsh.vector.*; -import ghidra.app.decompiler.DecompInterface; -import ghidra.app.decompiler.DecompileOptions; -import ghidra.app.decompiler.signature.SignatureResult; -import ghidra.app.script.GhidraScript; -import ghidra.app.services.ProgramManager; -import ghidra.features.bsim.query.GenSignatures; -import ghidra.program.model.address.Address; -import ghidra.program.model.lang.LanguageID; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; -import ghidra.util.xml.SpecXmlUtils; -import ghidra.xml.NonThreadedXmlPullParserImpl; -import ghidra.xml.XmlPullParser; - -public class CompareSignatures extends GhidraScript { - - private LSHVectorFactory vectorFactory; - - private LSHVector generateVector(Function f, Program program) { - DecompInterface decompiler = new DecompInterface(); - decompiler.setOptions(new DecompileOptions()); - decompiler.toggleSyntaxTree(false); - decompiler.setSignatureSettings(vectorFactory.getSettings()); - if (!decompiler.openProgram(program)) { - println("Unable to initalize the Decompiler interface"); - println(decompiler.getLastMessage()); - return null; - } - SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null); - LSHVector vec = vectorFactory.buildVector(sigres.features); - return vec; - } - - private Program getProgram(Program[] progarray, String name) { - if ((name == null) || (progarray == null)) { - return null; - } - for (Program prog : progarray) { - if (name.equals(prog.getName())) { - return prog; - } - } - return null; - } - - private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile) - throws FileNotFoundException, IOException, SAXException { - InputStream input = weightsFile.getInputStream(); - XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser", - SpecXmlUtils.getXmlHandler(), false); - vectorFactory.readWeights(parser); - input.close(); - } - - private void buildLSHVectorFactory() { - vectorFactory = new WeightedLSHCosineVectorFactory(); - try { - LanguageID id = currentProgram.getLanguageID(); - ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id); - readWeights(vectorFactory, defaultWeightsFile); - } - catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - catch (SAXException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - @Override - protected void run() throws Exception { - Function func = this.getFunctionContaining(this.currentAddress); - if (func == null) { - return; - } - buildLSHVectorFactory(); - LSHVector vec = generateVector(func, currentProgram); - ProgramManager programManager = state.getTool().getService(ProgramManager.class); - Program[] progarray = programManager.getAllOpenPrograms(); - String lastprogram_string = System.getProperty("ghidra.lastprogram"); - Program lastprogram = getProgram(progarray, lastprogram_string); - VectorCompare veccompare = new VectorCompare(); - if (lastprogram != null) { - String addrstring = System.getProperty("ghidra.lastaddress"); - if (addrstring != null) { - Address addr = lastprogram.getAddressFactory().getAddress(addrstring); - Function lastfunction = lastprogram.getFunctionManager().getFunctionAt(addr); - if (lastfunction != null) { - LSHVector lastvector = generateVector(lastfunction, lastprogram); - double sim = lastvector.compare(vec, veccompare); - double signif = vectorFactory.calculateSignificance(veccompare); - StringBuilder buf = new StringBuilder(); - buf.append("Comparison results:\n"); - buf.append(lastprogram.getName()); - buf.append("."); - buf.append(lastfunction.getName()); - buf.append(" vs. "); - buf.append(currentProgram.getName()); - buf.append("."); - buf.append(func.getName()); - buf.append("\n Similarity: "); - buf.append(Double.toString(sim)); - buf.append("\n Significance: "); - buf.append(Double.toString(signif)); - buf.append("\n"); - lastvector.compareDetail(vec, buf); - println(buf.toString()); - } - } - } - System.setProperty("ghidra.lastprogram", currentProgram.getName()); - String addrstring = func.getEntryPoint().toString(); - System.setProperty("ghidra.lastaddress", addrstring); - } - -} diff --git a/Ghidra/Features/BSim/ghidra_scripts/CompareSignaturesSpecifyWeights.java b/Ghidra/Features/BSim/ghidra_scripts/CompareSignaturesSpecifyWeights.java deleted file mode 100755 index 3886b2133e..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/CompareSignaturesSpecifyWeights.java +++ /dev/null @@ -1,155 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// Compare the BSim feature vectors of two functions. -//@category BSim - -import java.io.*; - -import org.xml.sax.SAXException; - -import generic.jar.ResourceFile; -import generic.lsh.vector.*; -import ghidra.app.decompiler.DecompInterface; -import ghidra.app.decompiler.DecompileOptions; -import ghidra.app.decompiler.signature.SignatureResult; -import ghidra.app.script.GhidraScript; -import ghidra.app.services.ProgramManager; -import ghidra.framework.Application; -import ghidra.program.model.address.Address; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; -import ghidra.util.exception.CancelledException; -import ghidra.util.xml.SpecXmlUtils; -import ghidra.xml.NonThreadedXmlPullParserImpl; -import ghidra.xml.XmlPullParser; - -public class CompareSignaturesSpecifyWeights extends GhidraScript { - - private static final String DEFAULT_LSH_WEIGHTS_FILE = "lshweights_nosize.xml"; - private LSHVectorFactory vectorFactory; - - private LSHVector generateVector(Function f, Program program) { - DecompInterface decompiler = new DecompInterface(); - decompiler.setOptions(new DecompileOptions()); - decompiler.setSignatureSettings(vectorFactory.getSettings()); - decompiler.toggleSyntaxTree(false); - if (!decompiler.openProgram(program)) { - println("Unable to initalize the Decompiler interface"); - println(decompiler.getLastMessage()); - return null; - } - - SignatureResult sigres = decompiler.generateSignatures(f, false, 10, null); - - LSHVector vec = vectorFactory.buildVector(sigres.features); - return vec; - } - - private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile) - throws FileNotFoundException, IOException, SAXException { - InputStream input = weightsFile.getInputStream(); - XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser", - SpecXmlUtils.getXmlHandler(), false); - vectorFactory.readWeights(parser); - input.close(); - } - - private boolean buildLSHVectorFactory() { - vectorFactory = new WeightedLSHCosineVectorFactory(); - try { - String weightsFile = - askString("Enter weights file name", "weights file", DEFAULT_LSH_WEIGHTS_FILE); - ResourceFile defaultWeightsFile = Application.findDataFileInAnyModule(weightsFile); - readWeights(vectorFactory, defaultWeightsFile); - } - catch (FileNotFoundException e) { - e.printStackTrace(); - return false; - } - catch (IOException e) { - e.printStackTrace(); - return false; - } - catch (SAXException e) { - e.printStackTrace(); - return false; - } - catch (CancelledException e) { - return false; - } - return true; - } - - private Program getProgram(Program[] progarray, String name) { - if ((name == null) || (progarray == null)) { - return null; - } - for (Program prog : progarray) { - if (name.equals(prog.getName())) { - return prog; - } - } - return null; - } - - @Override - protected void run() throws Exception { - Function func = this.getFunctionContaining(this.currentAddress); - if (func == null) { - return; - } - if (!buildLSHVectorFactory()) { - return; - } - LSHVector vec = generateVector(func, currentProgram); - ProgramManager programManager = state.getTool().getService(ProgramManager.class); - Program[] progarray = programManager.getAllOpenPrograms(); - String lastprogram_string = System.getProperty("ghidra.lastprogram"); - Program lastprogram = getProgram(progarray, lastprogram_string); - VectorCompare veccompare = new VectorCompare(); - if (lastprogram != null) { - String addrstring = System.getProperty("ghidra.lastaddress"); - if (addrstring != null) { - Address addr = lastprogram.getAddressFactory().getAddress(addrstring); - Function lastfunction = lastprogram.getFunctionManager().getFunctionAt(addr); - if (lastfunction != null) { - LSHVector lastvector = generateVector(lastfunction, lastprogram); - double sim = lastvector.compare(vec, veccompare); - double signif = vectorFactory.calculateSignificance(veccompare); - StringBuilder buf = new StringBuilder(); - buf.append("Comparison results:\n"); - buf.append(lastprogram.getName()); - buf.append("."); - buf.append(lastfunction.getName()); - buf.append(" vs. "); - buf.append(currentProgram.getName()); - buf.append("."); - buf.append(func.getName()); - buf.append("\n Similarity: "); - buf.append(Double.toString(sim)); - buf.append("\n Significance: "); - buf.append(Double.toString(signif)); - buf.append("\n"); - lastvector.compareDetail(vec, buf); - println(buf.toString()); - } - } - } - System.setProperty("ghidra.lastprogram", currentProgram.getName()); - String addrstring = func.getEntryPoint().toString(); - System.setProperty("ghidra.lastaddress", addrstring); - } -} diff --git a/Ghidra/Features/BSim/ghidra_scripts/DebugSignatures.java b/Ghidra/Features/BSim/ghidra_scripts/DebugSignatures.java deleted file mode 100755 index bcd2a2723f..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/DebugSignatures.java +++ /dev/null @@ -1,72 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.List; - -import ghidra.app.decompiler.DecompInterface; -import ghidra.app.decompiler.DecompileOptions; -import ghidra.app.decompiler.signature.DebugSignature; -import ghidra.app.script.GhidraScript; -import ghidra.program.model.lang.Language; -import ghidra.program.model.listing.Function; - -public class DebugSignatures extends GhidraScript { - - private static final int SIGNATURE_SETTINGS = 0x45; - - @Override - protected void run() throws Exception { - Function func = this.getFunctionContaining(this.currentAddress); - - if (func == null) { - popup("No function selected!"); - return; - } - - DecompInterface decompiler = new DecompInterface(); - decompiler.setOptions(new DecompileOptions()); - decompiler.toggleSyntaxTree(false); - decompiler.setSignatureSettings(SIGNATURE_SETTINGS); - if (!decompiler.openProgram(this.currentProgram)) { - println("Unable to initalize the Decompiler interface"); - println(decompiler.getLastMessage()); - return; - } - - Language language = this.currentProgram.getLanguage(); - List sigres = decompiler.debugSignatures(func, 10, null); - - StringBuffer buf = new StringBuffer(); - buf.append("\nFunction: "); - buf.append(func.getName()); - buf.append("\nentry: "); - buf.append(func.getEntryPoint().toString()); - buf.append("\n\n"); - if (sigres == null) { - printf("Null sigres!\n"); - } - else { - for (int i = 0; i < sigres.size(); ++i) { - sigres.get(i).printRaw(language, buf); - buf.append("\n"); - } - } - printf("%s\n", buf.toString()); - decompiler.closeProgram(); - decompiler.dispose(); - } - -} diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.java b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.java new file mode 100755 index 0000000000..eb585b75da --- /dev/null +++ b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.java @@ -0,0 +1,37 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generate the BSim signature for the function currently containing the cursor +// and dump the feature hashes and debug information to the console. +import java.util.List; + +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.signature.DebugSignature; +import ghidra.program.model.lang.Language; +import ghidra.program.model.listing.Function; + +public class DumpBSimDebugSignaturesScript extends DumpBSimSignaturesScript { + + @Override + protected void getSignatures(StringBuffer buf, DecompInterface decompiler, Function func) { + List sigres = decompiler.debugSignatures(func, 10, null); + Language language = currentProgram.getLanguage(); + for (int i = 0; i < sigres.size(); ++i) { + sigres.get(i).printRaw(language, buf); + buf.append("\n"); + } + } + +} diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.py b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.py new file mode 100755 index 0000000000..3d6221595a --- /dev/null +++ b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimDebugSignaturesScript.py @@ -0,0 +1,48 @@ +## ### +# IP: GHIDRA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +# Generate the BSim signature for the function at the current address, +# then dump the signature hashes and debug information to the console +# @category: BSim.python + +import ghidra.app.decompiler.DecompInterface as DecompInterface +import ghidra.app.decompiler.DecompileOptions as DecompileOptions + +def processFunction(func): + decompiler = DecompInterface() + try: + options = DecompileOptions() + decompiler.setOptions(options) + decompiler.toggleSyntaxTree(False) + decompiler.setSignatureSettings(0x4d) + if not decompiler.openProgram(currentProgram): + print "Unable to initialize the Decompiler interface!" + print "%s" % decompiler.getLastMessage() + return + language = currentProgram.getLanguage() + sigres = decompiler.debugSignatures(func,10,None) + for i,res in enumerate(sigres): + buf = java.lang.StringBuffer() + sigres.get(i).printRaw(language,buf) + print "%s" % buf.toString() + finally: + decompiler.closeProgram() + decompiler.dispose() + +func = currentProgram.getFunctionManager().getFunctionContaining(currentAddress) +if func is None: + print "no function at current address" +else: + processFunction(func) diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.java b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.java new file mode 100755 index 0000000000..dd764ac016 --- /dev/null +++ b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.java @@ -0,0 +1,80 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generate the BSim signature for the function currently +// containing the cursor and dump the feature hashes to the console. +//@category BSim + +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileOptions; +import ghidra.app.decompiler.signature.SignatureResult; +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.Function; + +public class DumpBSimSignaturesScript extends GhidraScript { + + protected static final int SIGNATURE_SETTINGS = 0x4d; + + @Override + public void run() throws Exception { + if (isRunningHeadless()) { + popup("This script must be run in the Ghidra GUI"); + return; + } + if (currentProgram == null) { + popup("This script requires an open program"); + return; + } + Function func = getFunctionContaining(currentAddress); + if (func == null) { + popup("No function selected!"); + return; + } + + DecompInterface decompiler = new DecompInterface(); + try { + decompiler.setOptions(new DecompileOptions()); + decompiler.toggleSyntaxTree(false); + decompiler.setSignatureSettings(SIGNATURE_SETTINGS); + if (!decompiler.openProgram(currentProgram)) { + println("Unable to initalize the Decompiler interface"); + println(decompiler.getLastMessage()); + return; + } + + StringBuffer buf = new StringBuffer(); + buf.append("\nFunction: "); + buf.append(func.getName()); + buf.append("\nentry: "); + buf.append(func.getEntryPoint().toString()); + buf.append("\n\n"); + getSignatures(buf, decompiler, func); + printf("%s\n", buf.toString()); + } + finally { + decompiler.closeProgram(); + decompiler.dispose(); + } + } + + protected void getSignatures(StringBuffer buf, DecompInterface decompiler, Function func) { + SignatureResult sigres = decompiler.generateSignatures(func, false, 10, null); + for (int feature : sigres.features) { + buf.append(Integer.toHexString(feature)); + buf.append("\n"); + } + } + +} diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpDebugSignatures.py b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.py similarity index 60% rename from Ghidra/Features/BSim/ghidra_scripts/DumpDebugSignatures.py rename to Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.py index 9877b8217d..c1189492cc 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/DumpDebugSignatures.py +++ b/Ghidra/Features/BSim/ghidra_scripts/DumpBSimSignaturesScript.py @@ -13,36 +13,38 @@ # See the License for the specific language governing permissions and # limitations under the License. ## -# Use the decompiler to generate signatures for the function at the current address, then dump the -# signature hashes and debug information to the console +# Generate the BSim signature for the function at the current address, then dump the +# signature hashes to the console # @category: BSim.python -import ghidra.app.decompiler.tracking.DecompInterfaceTracking as DecompInterfaceTracking +import ghidra.app.decompiler.DecompInterface as DecompInterface import ghidra.app.decompiler.DecompileOptions as DecompileOptions import generic.lsh.vector.WeightedLSHCosineVectorFactory as WeightedLSHCosineVectorFactory -import ghidra.query.GenSignatures as GenSignatures +import ghidra.features.bsim.query.GenSignatures as GenSignatures import ghidra.xml.NonThreadedXmlPullParserImpl as NonThreadedXmlPullParserImpl import ghidra.util.xml.SpecXmlUtils as SpecXmlUtils def processFunction(func): - decompiler = DecompInterfaceTracking() - options = DecompileOptions() - decompiler.setOptions(options) - decompiler.toggleSyntaxTree(False) - decompiler.setSignatureSettings(getSettings()) - if not decompiler.openProgram(currentProgram): - print "Unable to initialize the Decompiler interface!" - print "%s" % decompiler.getLastMessage() - return - language = currentProgram.getLanguage() - sigres = decompiler.debugSignatures(func,10,None) - for i,res in enumerate(sigres): + decompiler = ghidra.app.decompiler.DecompInterface() + try: + options = ghidra.app.decompiler.DecompileOptions() + decompiler.setOptions(options) + decompiler.toggleSyntaxTree(False) + decompiler.setSignatureSettings(getSettings()) + if not decompiler.openProgram(currentProgram): + print "Unable to initialize the Decompiler interface!" + print "%s" % decompiler.getLastMessage() + return + sigres = decompiler.generateSignatures(func, False, 10, None) buf = java.lang.StringBuffer() - sigres.get(i).printRaw(language,buf) - print "%s" % buf.toString() - decompiler.closeProgram() - decompiler.dispose() + for i,res in enumerate(sigres.features): + buf.append(java.lang.Integer.toHexString(sigres.features[i])) + buf.append("\n") + print buf.toString() + finally: + decompiler.closeProgram() + decompiler.dispose() def getSettings(): vectorFactory = WeightedLSHCosineVectorFactory() diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.java b/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.java deleted file mode 100755 index aad4ddc12b..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.java +++ /dev/null @@ -1,115 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// Use the decompiler to generate signatures for the function currently containing the cursor -// and dump the signature hashes to the console -//@category BSim - -import java.io.*; -import java.util.List; - -import org.xml.sax.SAXException; - -import generic.jar.ResourceFile; -import generic.lsh.vector.LSHVectorFactory; -import generic.lsh.vector.WeightedLSHCosineVectorFactory; -import ghidra.app.decompiler.DecompInterface; -import ghidra.app.decompiler.DecompileOptions; -import ghidra.app.decompiler.signature.DebugSignature; -import ghidra.app.decompiler.signature.SignatureResult; -import ghidra.app.script.GhidraScript; -import ghidra.features.bsim.query.GenSignatures; -import ghidra.program.model.lang.Language; -import ghidra.program.model.lang.LanguageID; -import ghidra.program.model.listing.Function; -import ghidra.util.xml.SpecXmlUtils; -import ghidra.xml.NonThreadedXmlPullParserImpl; -import ghidra.xml.XmlPullParser; - -public class DumpSignatures extends GhidraScript { - - private LSHVectorFactory vectorFactory; - - @Override - public void run() throws Exception { - Function func = this.getFunctionContaining(this.currentAddress); - if (func == null) { - return; - } - buildLSHVectorFactory(); - boolean debug = false; - DecompInterface decompiler = new DecompInterface(); - decompiler.setOptions(new DecompileOptions()); - decompiler.setSignatureSettings(vectorFactory.getSettings()); - decompiler.toggleSyntaxTree(false); - if (!decompiler.openProgram(this.currentProgram)) { - println("Unable to initalize the Decompiler interface"); - println(decompiler.getLastMessage()); - return; - } - if (!debug) { - SignatureResult sigres = decompiler.generateSignatures(func, false, 10, null); - StringBuffer buf = new StringBuffer("\n"); - for (int feature : sigres.features) { - buf.append(Integer.toHexString(feature)); - buf.append("\n"); - } - println(buf.toString()); - } - else { - Language language = this.currentProgram.getLanguage(); - List sigres = decompiler.debugSignatures(func, 10, null); - StringBuffer buf = new StringBuffer("\n"); - for (int i = 0; i < sigres.size(); ++i) { - sigres.get(i).printRaw(language, buf); - buf.append("\n"); - } - println(buf.toString()); - } - decompiler.closeProgram(); - decompiler.dispose(); - } - - private static void readWeights(LSHVectorFactory vectorFactory, ResourceFile weightsFile) - throws FileNotFoundException, IOException, SAXException { - InputStream input = weightsFile.getInputStream(); - XmlPullParser parser = new NonThreadedXmlPullParserImpl(input, "Vector weights parser", - SpecXmlUtils.getXmlHandler(), false); - vectorFactory.readWeights(parser); - input.close(); - } - - private void buildLSHVectorFactory() { - vectorFactory = new WeightedLSHCosineVectorFactory(); - try { - LanguageID id = currentProgram.getLanguageID(); - ResourceFile defaultWeightsFile = GenSignatures.getWeightsFile(id, id); - readWeights(vectorFactory, defaultWeightsFile); - } - catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - catch (SAXException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - -} diff --git a/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.py b/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.py deleted file mode 100755 index 1f336303aa..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/DumpSignatures.py +++ /dev/null @@ -1,61 +0,0 @@ -## ### -# IP: GHIDRA -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## -# Use the decompiler to generate signatures for the function at the current address, then dump the -# signature hashes to the console -# @category: BSim.python - -import ghidra.app.decompiler.tracking.DecompInterfaceTracking as DecompInterfaceTracking -import ghidra.app.decompiler.DecompileOptions as DecompileOptions -import generic.lsh.vector.WeightedLSHCosineVectorFactory as WeightedLSHCosineVectorFactory -import ghidra.query.GenSignatures as GenSignatures -import ghidra.xml.NonThreadedXmlPullParserImpl as NonThreadedXmlPullParserImpl -import ghidra.util.xml.SpecXmlUtils as SpecXmlUtils - - -def processFunction(func): - decompiler = ghidra.app.decompiler.tracking.DecompInterfaceTracking() - options = ghidra.app.decompiler.DecompileOptions() - decompiler.setOptions(options) - decompiler.toggleSyntaxTree(False) - decompiler.setSignatureSettings(getSettings()) - if not decompiler.openProgram(currentProgram): - print "Unable to initialize the Decompiler interface!" - print "%s" % decompiler.getLastMessage() - return - sigres = decompiler.generateSignatures(func, False, 10, None) - buf = java.lang.StringBuffer() - for i,res in enumerate(sigres.features): - buf.append(java.lang.Integer.toHexString(sigres.features[i])) - buf.append("\n") - print buf.toString() - decompiler.closeProgram() - decompiler.dispose() - -def getSettings(): - vectorFactory = WeightedLSHCosineVectorFactory() - id = currentProgram.getLanguageID() - defaultWeightsFile = GenSignatures.getWeightsFile(id,id) - input = defaultWeightsFile.getInputStream() - parser = NonThreadedXmlPullParserImpl(input,"Vector weights parser", SpecXmlUtils.getXmlHandler(),False) - vectorFactory.readWeights(parser) - input.close() - return vectorFactory.getSettings() - -func = currentProgram.getFunctionManager().getFunctionContaining(currentAddress) -if func is None: - print "no function at current address" -else: - processFunction(func) diff --git a/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.java b/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.java similarity index 89% rename from Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.java rename to Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.java index 6b510930ef..657e8ef0ef 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.java +++ b/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.java @@ -26,24 +26,22 @@ import ghidra.features.bsim.query.protocol.SimilarityVectorResult; import ghidra.program.database.symbol.FunctionSymbol; import ghidra.program.model.listing.*; +public class ExampleOverviewQueryScript extends GhidraScript { + private static final double SIMILARITY_BOUND = 0.7; + private static final double SIGNIFICANCE_BOUND = 0.0; -public class ExampleOverviewQuery extends GhidraScript { - private static final double SIMILARITY_BOUND = 0.7; - private static final double SIGNIFICANCE_BOUND = 0.0; - - @Override protected void run() throws Exception { Program queryingProgram = currentProgram; HashSet funcsToQuery = new HashSet<>(); FunctionIterator fIter = queryingProgram.getFunctionManager().getFunctionsNoStubs(true); - for (Function func : fIter){ + for (Function func : fIter) { funcsToQuery.add((FunctionSymbol) func.getSymbol()); - } + } SFOverviewInfo overviewInfo = new SFOverviewInfo(funcsToQuery); - overviewInfo.setSimilarityThreshold(SIMILARITY_BOUND); + overviewInfo.setSimilarityThreshold(SIMILARITY_BOUND); overviewInfo.setSignificanceThreshold(SIGNIFICANCE_BOUND); - + try (SimilarFunctionQueryService queryService = new SimilarFunctionQueryService(queryingProgram)) { String DATABASE_URL = askString("Enter database URL", "URL:"); @@ -65,5 +63,4 @@ public class ExampleOverviewQuery extends GhidraScript { printf("%s\n", buf.toString()); } } - } diff --git a/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.py b/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.py similarity index 89% rename from Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.py rename to Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.py index 5a7f4751d3..0e3877d167 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQuery.py +++ b/Ghidra/Features/BSim/ghidra_scripts/ExampleOverviewQueryScript.py @@ -16,8 +16,8 @@ # Example of how to perform an overview query in a script # @category BSim.python -import ghidra.query.facade.SFOverviewInfo as SFOverviewInfo -import ghidra.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService +import ghidra.features.bsim.query.facade.SFOverviewInfo as SFOverviewInfo +import ghidra.features.bsim.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService import java.util.HashSet SIMILARITY_BOUND = 0.7 @@ -37,7 +37,7 @@ DB_URL = askString("Enter database URL", "URL:") queryService.initializeDatabase(DB_URL) vectorFactory = queryService.getLSHVectorFactory() -overviewResults = queryService.overviewSimilarFunctions(overviewInfo, monitor) +overviewResults = queryService.overviewSimilarFunctions(overviewInfo, None, monitor) for result in overviewResults.result: print "Name: %s" % result.getBase().getFunctionName() diff --git a/Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClient.java b/Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClientScript.java similarity index 61% rename from Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClient.java rename to Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClientScript.java index 5ee7eab373..cc5fc75129 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClient.java +++ b/Ghidra/Features/BSim/ghidra_scripts/ExampleQueryClientScript.java @@ -20,18 +20,51 @@ import java.io.StringWriter; import java.net.URL; import java.util.List; +import org.apache.commons.lang3.StringUtils; + import ghidra.app.script.GhidraScript; -import ghidra.features.bsim.query.BSimClientFactory; -import ghidra.features.bsim.query.FunctionDatabase; +import ghidra.features.base.values.GhidraValuesMap; +import ghidra.features.bsim.query.*; import ghidra.features.bsim.query.description.*; import ghidra.features.bsim.query.protocol.*; +import ghidra.util.MessageType; import ghidra.util.Msg; -public class ExampleQueryClient extends GhidraScript { +public class ExampleQueryClientScript extends GhidraScript { + + private static final String URL = "URL"; + private static final String EXECUTABLE_NAME = "Executable Name"; + private static final String FUNCTION_NAME = "Function Name"; @Override protected void run() throws Exception { - URL url = BSimClientFactory.deriveBSimURL("ghidra://localhost/repo"); + GhidraValuesMap values = new GhidraValuesMap(); + values.defineString(URL); + values.defineString(EXECUTABLE_NAME); + values.defineString(FUNCTION_NAME); + + values.setValidator((valueMap, status) -> { + String url = valueMap.getString(URL); + if (StringUtils.isBlank(url)) { + status.setStatusText(URL + " cannot be empty!", MessageType.ERROR); + return false; + } + String exe = valueMap.getString(EXECUTABLE_NAME); + if (StringUtils.isBlank(exe)) { + status.setStatusText(EXECUTABLE_NAME + " cannot be empty!", MessageType.ERROR); + return false; + } + String func = valueMap.getString(FUNCTION_NAME); + if (StringUtils.isBlank(func)) { + status.setStatusText(FUNCTION_NAME + " cannot be empty!", MessageType.ERROR); + return false; + } + return true; + }); + + askValues("BSim Query Info", "BSim Query Info", values); + + URL url = BSimClientFactory.deriveBSimURL(values.getString(URL)); try (FunctionDatabase client = BSimClientFactory.buildClient(url, false)) { if (!client.initialize()) { Msg.error(this, "Unable to connect to server"); @@ -45,15 +78,16 @@ public class ExampleQueryClient extends GhidraScript { write.flush(); QueryName exequery = new QueryName(); - exequery.spec.exename = "libdocdoxygenplugin.so"; + exequery.spec.exename = values.getString(EXECUTABLE_NAME); ResponseName respname = exequery.execute(client); if (respname == null) { Msg.error(this, client.getLastError()); return; } + ExecutableRecord erec = respname.manage.getExecutableRecordSet().first(); FunctionDescription funcrec = - respname.manage.findFunctionByName("DocDoxygenPlugin::createCatalog", erec); + respname.manage.findFunctionByName(values.getString(FUNCTION_NAME), erec); QueryChildren childquery = new QueryChildren(); childquery.md5sum = funcrec.getExecutableRecord().getMd5(); @@ -76,7 +110,7 @@ public class ExampleQueryClient extends GhidraScript { } } write.flush(); - Msg.info(this, write.toString()); + printf("%s", write.toString()); } } diff --git a/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.java b/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.java index ec28acfebc..206102b0a3 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.java +++ b/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.java @@ -50,24 +50,31 @@ public class GenerateSignatures extends GhidraScript { final LSHVectorFactory vectorFactory = FunctionDatabase.generateLSHVectorFactory(); final GenSignatures gensig = new GenSignatures(true); - final String templatename = - askString("GenerateSignatures:", "Database template", "medium_nosize"); - final Configuration config = FunctionDatabase.loadConfigurationTemplate(templatename); - vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings); - gensig.setVectorFactory(vectorFactory); - gensig.addExecutableCategories(config.info.execats); - gensig.addFunctionTags(config.info.functionTags); - gensig.addDateColumnName(config.info.dateColumnName); - final String repo = "ghidra://localhost/" + state.getProject().getName(); - final String path = GenSignatures.getPathFromDomainFile(currentProgram); - gensig.openProgram(this.currentProgram, null, null, null, repo, path); - final FunctionManager fman = currentProgram.getFunctionManager(); - final Iterator iter = fman.getFunctions(true); - gensig.scanFunctions(iter, fman.getFunctionCount(), monitor); - final FileWriter fwrite = new FileWriter(file); - final DescriptionManager manager = gensig.getDescriptionManager(); - manager.saveXml(fwrite); - fwrite.close(); + try { + final String templatename = + askString("GenerateSignatures:", "Database template", "medium_nosize"); + final Configuration config = FunctionDatabase.loadConfigurationTemplate(templatename); + vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings); + gensig.setVectorFactory(vectorFactory); + gensig.addExecutableCategories(config.info.execats); + gensig.addFunctionTags(config.info.functionTags); + gensig.addDateColumnName(config.info.dateColumnName); + final String repo = "ghidra://localhost/" + state.getProject().getName(); + final String path = GenSignatures.getPathFromDomainFile(currentProgram); + gensig.openProgram(this.currentProgram, null, null, null, repo, path); + final FunctionManager fman = currentProgram.getFunctionManager(); + final Iterator iter = fman.getFunctions(true); + gensig.scanFunctions(iter, fman.getFunctionCount(), monitor); + try (FileWriter fwrite = new FileWriter(file)) { + final DescriptionManager manager = gensig.getDescriptionManager(); + manager.saveXml(fwrite); + fwrite.close(); + } + } + finally { + gensig.dispose(); + } + } } diff --git a/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.py b/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.py index becc8233ce..4cbc1edbbb 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.py +++ b/Ghidra/Features/BSim/ghidra_scripts/GenerateSignatures.py @@ -18,8 +18,8 @@ import java.lang.System as System import java.io.File as File -import ghidra.query.FunctionDatabase as FunctionDatabase -import ghidra.query.GenSignatures as GenSignatures +import ghidra.features.bsim.query.FunctionDatabase as FunctionDatabase +import ghidra.features.bsim.query.GenSignatures as GenSignatures import java.io.FileWriter as FileWriter def run(): diff --git a/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.java b/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.java index 2b69dc680b..da4cd95b33 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.java +++ b/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.java @@ -25,27 +25,24 @@ import ghidra.features.bsim.query.description.*; import ghidra.features.bsim.query.protocol.*; import ghidra.program.model.listing.Function; - public class QueryFunction extends GhidraScript { - //GenSignatures gensig; - //FunctionDatabase database; private static final int MATCHES_PER_FUNC = 10; private static final double SIMILARITY_BOUND = 0.7; private static final double CONFIDENCE_BOUND = 0.0; - + @Override public void run() throws Exception { if (currentProgram == null) { return; } - Function func = this.getFunctionContaining(this.currentAddress); - if (func == null){ - popup("No function selected!"); - return; - } - - String DATABASE_URL = askString("Enter Database URL", "URL"); + Function func = this.getFunctionContaining(this.currentAddress); + if (func == null) { + popup("No function selected!"); + return; + } + + String DATABASE_URL = askString("Enter Database URL", "URL"); URL url = BSimClientFactory.deriveBSimURL(DATABASE_URL); try (FunctionDatabase database = BSimClientFactory.buildClient(url, false)) { if (!database.initialize()) { diff --git a/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.py b/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.py index 38a16768b6..9393b795dd 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.py +++ b/Ghidra/Features/BSim/ghidra_scripts/QueryFunction.py @@ -16,9 +16,9 @@ # Example of performing a BSim query on a single function # @category BSim.python -import ghidra.query.BSimClientFactory as BSimClientFactory -import ghidra.query.GenSignatures as GenSignatures -import ghidra.query.protocol.QueryNearest as QueryNearest +import ghidra.features.bsim.query.BSimClientFactory as BSimClientFactory +import ghidra.features.bsim.query.GenSignatures as GenSignatures +import ghidra.features.bsim.query.protocol.QueryNearest as QueryNearest MATCHES_PER_FUNC = 100 SIMILARITY_BOUND = 0.7 diff --git a/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.java b/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.java index 508f2423dc..73d6f5a103 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.java +++ b/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.java @@ -123,7 +123,7 @@ public class QueryWithFiltersScript extends GhidraScript { // the items are "AND'd together. // // ie: "The compiler cannot equal windows AND the compiler cannot equal foo_compiler". - addBsimFilter(new CompilerBSimFilterType(), "windows, foo_compiler"); + addBsimFilter(new NotCompilerBSimFilterType(), "windows, foo_compiler"); //connect to the database try { @@ -149,8 +149,7 @@ public class QueryWithFiltersScript extends GhidraScript { // set returned from the previous query. addBsimFilter(new Md5BSimFilterType(), currentProgram.getExecutableMD5()); addBsimFilter(new CompilerBSimFilterType(), "gcc"); - addBsimFilter(new FunctionTagBSimFilterType("KNOWN_LIBRARY", queryService), - "false"); + //addBsimFilter(new FunctionTagBSimFilterType("KNOWN_LIBRARY", queryService), "false"); // Apply the filters and print results. List filteredRows = @@ -194,7 +193,7 @@ public class QueryWithFiltersScript extends GhidraScript { * @throws CancelledException if the user cancelled the operation */ private List executeQuery(SFQueryInfo qInfo) - throws QueryDatabaseException, CancelledException { + throws QueryDatabaseException, CancelledException { SFQueryResult queryResults = queryService.querySimilarFunctions(qInfo, null, monitor); List resultRows = @@ -228,8 +227,8 @@ public class QueryWithFiltersScript extends GhidraScript { @Override public boolean test(Program t, FunctionDescription u) { return queryService.getLSHVectorFactory() - .getSelfSignificance( - u.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND; + .getSelfSignificance( + u.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND; } }); @@ -238,9 +237,10 @@ public class QueryWithFiltersScript extends GhidraScript { // Filters out any functions with a self significance less than a // certain value. // - preFilter.addPredicate((x, y) -> queryService.getLSHVectorFactory() - .getSelfSignificance( - y.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND); + preFilter.addPredicate((x, + y) -> queryService.getLSHVectorFactory() + .getSelfSignificance( + y.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND); // // Option 3. Static method @@ -276,7 +276,7 @@ public class QueryWithFiltersScript extends GhidraScript { program.getAddressFactory().getDefaultAddressSpace().getAddress(funcDesc.getAddress()); Function function = program.getFunctionManager().getFunctionAt(address); - if (function == null || function.getName().equals(funcDesc.getFunctionName())) { + if (function == null || !function.getName().equals(funcDesc.getFunctionName())) { return false; } return function.getSymbol().getSource() != SourceType.ANALYSIS; diff --git a/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.py b/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.py deleted file mode 100755 index 00eaafe847..0000000000 --- a/Ghidra/Features/BSim/ghidra_scripts/QueryWithFiltersScript.py +++ /dev/null @@ -1,173 +0,0 @@ -## ### -# IP: GHIDRA -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## -# Advanced example of BSim querying -# @category BSim.python - -import ghidra.query.facade.SimilarFunctionQueryService as SimilarFunctionQueryService -import ghidra.query.facade.SFQueryInfo as SFQueryInfo -import ghidra.query.FunctionDatabase as FunctionDatabase -import ghidra.query.facade.QueryDatabaseException as QueryDatabaseException -import java.util.HashSet as HashSet -import ghidra.app.plugin.core.query.QueryNearestRow as QueryNearestRow -import java.util.function.BiPredicate as BiPredicate -import ghidra.query.protocol.FilterTemplate as FilterTemplate -import ghidra.app.plugin.core.query.ExecutableResult as ExecutableResult -import java.util.Comparator as Comparator -import java.util.Arrays as Arrays -import java.lang.Double as Double - -#Query thresholds -MAX_NUM_FUNCTIONS = 100 -SIMILARITY_BOUND = 0.7 -SIGNIFICANCE_BOUND = 0.0 - -#limit the number of results displayed -NUM_EXES_TO_DISPLAY = 10 - -#for prefiltering: this number will be used to filter out small functions -SELF_SIGNIFICANCE_BOUND = 40.0 - -def run(): - - #get the set of functions to query - funcsToQuery = getFunctionsToQuery() - - #sets up the object required for querying the database - queryService = SimilarFunctionQueryService(currentProgram) - queryInfo = SFQueryInfo(funcsToQuery) - bsimFilter = queryInfo.getBsimFilter() - - #sets the query parameters. - #change the defined constants to control how fuzzy of - #a match you're willing to accept, and the maximum number - #of matches to return for each function - queryInfo.setMaximumResults(MAX_NUM_FUNCTIONS) - queryInfo.setSimilarityThreshold(SIMILARITY_BOUND) - queryInfo.setSignificanceThreshold(SIGNIFICANCE_BOUND) - - #add the prefilters - setPrefilters(queryService, queryInfo) - - #add a filter on the date - addBsimFilter(bsimFilter, FilterTemplate.DateLater(""), "01/01/1776") - - #add a filter with multiple values. Since this is an "Equal" filter, the results are OR'd together - #so a given executable will pass the main filter if it passes at least one of the subfilters - addBsimFilter(bsimFilter, FilterTemplate.ArchEquals(),"x86:LE:64:default, x86:LE:32:default, ARM:LE:32:v4") - - #now add a "notequal" filter - #to pass, the compiler can't be windows and it can't be foo_compiler - addBsimFilter(bsimFilter,FilterTemplate.CompNotEqual(),"windows, foo_compiler") - - #establish a connection to the BSim database - try: - dbUrl = askString("","Enter the URL of the BSim database:", "ghidra://localhost/bsimDB") - queryService.initializeDatabase(dbUrl) - error = queryService.getDatabase().getLastError() - if error is not None and (error.category is ErrorCategory.Nodatabase): - print "Database [%s] cannot be found (does it exist?)" % dbUrl - return - except QueryDatabaseException as e: - print e.getMessage() - return - - resultRows = executeQuery(queryService,queryInfo) - printFunctionQueryResults(resultRows, "\nFunction-level results before filtering") - - #now add some post-query filters, which filters the result set returned by the previous query - - addBsimFilter(bsimFilter, FilterTemplate.Md5NotEqual(), currentProgram.getExecutableMD5()) - addBsimFilter(bsimFilter, FilterTemplate.CompilerEquals(), "gcc") - addBsimFilter(bsimFilter, FilterTemplate.FunctionTagTemplate("KNOWN_LIBRARY", queryService), "false") - - #apply the filters and print the results - filteredRows = QueryNearestRow.filterMatchRows(bsimFilter, resultRows) - printFunctionQueryResults(filteredRows, "\nFunction-level results after filtering") - printExecutableInformation(filteredRows) - return - - -#collect the functions to query from currentProgram -def getFunctionsToQuery(): - functions = HashSet(); - fIter = currentProgram.getFunctionManager().getFunctionsNoStubs(True) - for func in fIter: - functions.add(func.getSymbol()) - return functions - -#query the database -def executeQuery(queryService,queryInfo): - queryResults = queryService.querySimilarFunctions(queryInfo,monitor) - resultRows = QueryNearestRow.generate(queryResults.getSimilarityResults(),currentProgram) - return resultRows - -def printFunctionQueryResults(resultRows, title): - print "%s: %d\n\n" % (title, resultRows.size()) - for row in resultRows: - print " queried function: %s" % row.getOriginalFunctionDescription().getFunctionName() - print " matching function: %s" % row.getMatchFunctionDescription().getFunctionName() - print " executable of matching function: %s" % row.getMatchFunctionDescription().getExecutableRecord().getNameExec() - print " similarity: %f" % row.getSimilarity() - print " significance: %f\n" % row.getSignificance() - -#Prefilters are used to filter out functions before sending a query to the database -#A typical use case would be to collect all functions in a binary, then use a -#prefilter to remove the functions with low self-significance (which is the -#"BSim way" to remove small functions) -def setPrefilters(queryService, queryInfo): - preFilter = queryInfo.getPreFilter(); - selfSigFilter = ExampleFilter(queryService) - preFilter.addPredicate(selfSigFilter) - -class ExampleFilter(BiPredicate): - - def __init__(self, queryService): - self.queryService = queryService - - def test(self,program, fdesc): - return self.queryService.getLSHVectorFactory().getSelfSignificance(fdesc.getSignatureRecord().getLSHVector()) >= SELF_SIGNIFICANCE_BOUND - -def addBsimFilter(bsimFilter, filterTemplate, values): - for value in values.split(","): - if len(value.strip()) > 0: - bsimFilter.addAtom(filterTemplate, value.strip(), FilterTemplate.Blank()) - -#calls the methods to aggregate executable-level information about the matches -def printExecutableInformation(filteredRows): - execrows = ExecutableResult.generateFromMatchRows(filteredRows) - results = execrows.toArray() - sorter = Sorter() - Arrays.sort(results,sorter) - print "Executable-level results:" - numExes = min(len(results),NUM_EXES_TO_DISPLAY) - for i in range (numExes): - print " MD5: %s" % results[i].getExecutableRecord().getMd5() - print " Executable Name: %s" % results[i].getExecutableRecord().getNameExec() - print " Function Count: %d" % results[i].getFunctionCount() - print " Significance Sum: %f\n" % results[i].getSignificanceSum() - return - -class Sorter(Comparator): - - def __init__(self): - return - - def compare(self,o1,o2): - return Double.compare(o2.getSignificanceSum(), o1.getSignificanceSum()) - - - -run() diff --git a/Ghidra/Features/BSim/ghidra_scripts/TailoredAnalysis.java b/Ghidra/Features/BSim/ghidra_scripts/TailoredAnalysis.java index 06442edca5..90e84590cb 100755 --- a/Ghidra/Features/BSim/ghidra_scripts/TailoredAnalysis.java +++ b/Ghidra/Features/BSim/ghidra_scripts/TailoredAnalysis.java @@ -21,13 +21,14 @@ import ghidra.program.model.listing.Program; // suitable for BSim ingest process. Intended to be invoked as an analyzeHeadless -preScript //@category BSim +//note: script mentioned in BSim documentation by name public class TailoredAnalysis extends GhidraScript { @Override public void run() throws Exception { Options pl = currentProgram.getOptions(Program.ANALYSIS_PROPERTIES); pl.setBoolean("Decompiler Parameter ID", false); - + // These analyzers generate lots of cross references, which are not necessary for // signature analysis, and take time to run. On the other hand, you may want // them in general to facilitate general analysis @@ -42,9 +43,15 @@ public class TailoredAnalysis extends GhidraScript { // analyzerOptions.setBoolean("Commit Function Signatures", false); // You really want these options turned on - pl.setBoolean("Shared Return Calls",true); - pl.setBoolean("Function Start Search", true); - pl.setBoolean("DWARF", false); + pl.setBoolean("Shared Return Calls", true); + pl.setBoolean("Function Start Search", true); + + //The DWARF analyzer can take a long time, so for mass ingest it might be worth + //turning it off + //Moreover, the DWARF analyzer can lock prototypes, which can change the BSim signature + //of a function. This can negatively impact matches between executables with DWARF + //and executables without it. + pl.setBoolean("DWARF", false); // Options analyzerOptions = pl.getOptions("Function Start Search"); // analyzerOptions.setBoolean("Search Data Blocks", true); // analyzerOptions = pl.getOptions("Function Start Search After Code");