From 4592ebcd5a5b0a86430fe7600b944a25b9fb8589 Mon Sep 17 00:00:00 2001 From: dragonmacher <48328597+dragonmacher@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:24:05 -0400 Subject: [PATCH] GP-3195 - Updated ParallelDecompiler to not save processed items for cal that does not used them --- .../parallel/ParallelDecompiler.java | 46 +++++++++---------- .../app/util/DecompilerConcurrentQ.java | 41 +++++++++-------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/parallel/ParallelDecompiler.java b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/parallel/ParallelDecompiler.java index c296953336..a159f311a1 100644 --- a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/parallel/ParallelDecompiler.java +++ b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/parallel/ParallelDecompiler.java @@ -18,8 +18,7 @@ package ghidra.app.decompiler.parallel; import java.util.*; import java.util.function.Consumer; -import generic.concurrent.QCallback; -import generic.concurrent.QResult; +import generic.concurrent.*; import ghidra.app.util.DecompilerConcurrentQ; import ghidra.program.model.address.AddressSetView; import ghidra.program.model.listing.*; @@ -31,8 +30,8 @@ public class ParallelDecompiler { /** * Decompile the given functions using multiple decompilers - * - * @param callback the callback to be called for each that is processed + * + * @param callback the callback to be called for each item that is processed * @param program the program * @param addresses the addresses restricting which functions to decompile * @param monitor the task monitor @@ -53,8 +52,8 @@ public class ParallelDecompiler { /** * Decompile the given functions using multiple decompilers - * - * @param callback the callback to be called for each that is processed + * + * @param callback the callback to be called for each item that is processed * @param functions the functions to decompile * @param monitor the task monitor * @return the list of client results @@ -62,41 +61,40 @@ public class ParallelDecompiler { * @throws Exception if any other exception occurs */ public static List decompileFunctions(QCallback callback, - Collection functions, - TaskMonitor monitor) + Collection functions, TaskMonitor monitor) throws InterruptedException, Exception { List results = - doDecompileFunctions(callback, functions.iterator(), functions.size(), - monitor); + doDecompileFunctions(callback, functions.iterator(), functions.size(), monitor); return results; } /** * Decompile the given functions using multiple decompilers. - * + * *

Results will be passed to the given consumer as they are produced. Calling this - * method allows you to handle results as they are discovered. - * + * method allows you to handle results as they are discovered. + * *

This method will wait for all processing before returning. - * + * * @param callback the callback to be called for each that is processed - * @param program the program + * @param program the program * @param functions the functions to decompile * @param resultsConsumer the consumer to which results will be passed * @param monitor the task monitor * @throws InterruptedException if interrupted * @throws Exception if any other exception occurs */ - public static void decompileFunctions(QCallback callback, - Program program, - Iterator functions, Consumer resultsConsumer, - TaskMonitor monitor) + public static void decompileFunctions(QCallback callback, Program program, + Iterator functions, Consumer resultsConsumer, TaskMonitor monitor) throws InterruptedException, Exception { int max = program.getFunctionManager().getFunctionCount(); + boolean collectResults = false; // the client will process results as they arrive + GThreadPool threadPool = GThreadPool.getSharedThreadPool(THREAD_POOL_NAME); DecompilerConcurrentQ queue = - new DecompilerConcurrentQ<>(callback, THREAD_POOL_NAME, monitor); + new DecompilerConcurrentQ<>(callback, threadPool, collectResults, monitor); + monitor.initialize(max); queue.process(functions, resultsConsumer); queue.waitUntilDone(); @@ -130,14 +128,14 @@ public class ParallelDecompiler { } /** - * Creates an object that can be used to perform decompilation of a limited number of + * Creates an object that can be used to perform decompilation of a limited number of * functions at a time, as opposed to working over an entire range of functions at once. * {@link #decompileFunctions(QCallback, Program, AddressSetView, TaskMonitor)} will create * and tear down concurrent data structures on each use, making repeated calls less efficient. - * You would use this method when you wish to perform periodic work as results are returned + * You would use this method when you wish to perform periodic work as results are returned * and when using the callback mechanism is not sufficient such as when ordering of - * results is required. - * + * results is required. + * * @param callback the callback required to perform work. * @param monitor the monitor used to report progress and to cancel * @return the parallel decompiler used for decompiling. diff --git a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/util/DecompilerConcurrentQ.java b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/util/DecompilerConcurrentQ.java index ab9450e7c0..3cca028fe6 100644 --- a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/util/DecompilerConcurrentQ.java +++ b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/util/DecompilerConcurrentQ.java @@ -28,14 +28,14 @@ import utility.function.Dummy; /** * A class to perform some of the boilerplate setup of the {@link ConcurrentQ} that is shared * amongst clients that perform decompilation in parallel. - * - *

This class can be used in a blocking or non-blocking fashion. - * + * + *

This class can be used in a blocking or non-blocking fashion. + * *

    *
  • For blocking usage, call - * one of the {@code add} methods to put items in the queue and then call - * {@link #waitForResults()}.
    • - *
  • For non-blocking usage, simply call + * one of the {@code add} methods to put items in the queue and then call + * {@link #waitForResults()}.
    • + *
  • For non-blocking usage, simply call * {@link #process(Iterator, Consumer)}, passing the consumer of the results.
    • * *

    @@ -50,22 +50,23 @@ public class DecompilerConcurrentQ { private Consumer resultConsumer = Dummy.consumer(); public DecompilerConcurrentQ(QCallback callback, TaskMonitor monitor) { - this(callback, AutoAnalysisManager.getSharedAnalsysThreadPool(), monitor); + this(callback, AutoAnalysisManager.getSharedAnalsysThreadPool(), true, monitor); } public DecompilerConcurrentQ(QCallback callback, String threadPoolName, TaskMonitor monitor) { - this(callback, GThreadPool.getSharedThreadPool(threadPoolName), monitor); + this(callback, GThreadPool.getSharedThreadPool(threadPoolName), true, monitor); } - private DecompilerConcurrentQ(QCallback callback, GThreadPool pool, TaskMonitor monitor) { + public DecompilerConcurrentQ(QCallback callback, GThreadPool pool, boolean collectResults, + TaskMonitor monitor) { // @formatter:off queue = new ConcurrentQBuilder() - .setCollectResults(true) + .setCollectResults(collectResults) .setThreadPool(pool) .setMonitor(monitor) .setListener(new InternalResultListener()) - .build(callback); + .build(callback); // @formatter:on } @@ -84,7 +85,7 @@ public class DecompilerConcurrentQ { /** * Adds all items to the queue for processing. The results will be passed to the given consumer * as they are produced. - * + * * @param functions the functions to process * @param consumer the results consumer */ @@ -96,7 +97,7 @@ public class DecompilerConcurrentQ { /** * Waits for all results to be delivered. The client is responsible for processing the * results and handling any exceptions that may have occurred. - * + * * @return all results * @throws InterruptedException if interrupted while waiting */ @@ -110,10 +111,10 @@ public class DecompilerConcurrentQ { } /** - * Waits for all work to finish. Any exception encountered will trigger all processing to - * stop. If you wish for the work to continue despite exceptions, then use + * Waits for all work to finish. Any exception encountered will trigger all processing to + * stop. If you wish for the work to continue despite exceptions, then use * {@link #waitForResults()}. - * + * * @throws InterruptedException if interrupted while waiting * @throws Exception any exception that is encountered while processing items. */ @@ -131,9 +132,9 @@ public class DecompilerConcurrentQ { } /** - * Calls dispose on the queue being processed. Further, the call will block for up to - * timeoutSeconds while waiting for the queue to finish processing. - * + * Calls dispose on the queue being processed. Further, the call will block for up to + * timeoutSeconds while waiting for the queue to finish processing. + * * @param timeoutSeconds the number of seconds to wait for the disposed queue to finish * processing */ @@ -164,7 +165,7 @@ public class DecompilerConcurrentQ { } } catch (Throwable t) { - // This code is an asynchronous callback. Handle the exception the same way as + // This code is an asynchronous callback. Handle the exception the same way as // the waitXyz() method do, which is to shutdown the queue. Msg.error(this, "Unexpected exception getting Decompiler result", t); queue.dispose();