diff --git a/Ghidra/Features/Base/certification.manifest b/Ghidra/Features/Base/certification.manifest index 344f7c7936..292a996e04 100644 --- a/Ghidra/Features/Base/certification.manifest +++ b/Ghidra/Features/Base/certification.manifest @@ -36,6 +36,17 @@ data/parserprofiles/vs12Local.prf||GHIDRA||||END| data/pcodetest/EmuTesting.gdt||GHIDRA||||END| data/stringngrams/StringModel.sng||GHIDRA||reviewed||END| data/symbols/README.txt||GHIDRA||||END| +data/symbols/win16/commctrl.exports||GHIDRA||||END| +data/symbols/win16/commdlg.exports||GHIDRA||||END| +data/symbols/win16/gdi.exports||GHIDRA||||END| +data/symbols/win16/kernel.exports||GHIDRA||||END| +data/symbols/win16/krnl386.exports||GHIDRA||||END| +data/symbols/win16/lzexpand.exports||GHIDRA||||END| +data/symbols/win16/mmsystem.exports||GHIDRA||||END| +data/symbols/win16/toolhelp.exports||GHIDRA||||END| +data/symbols/win16/user.exports||GHIDRA||||END| +data/symbols/win16/ver.exports||GHIDRA||||END| +data/symbols/win16/win87em.exports||GHIDRA||||END| data/symbols/win32/kernel32.hints||GHIDRA||||END| data/symbols/win32/mfc100.exports||GHIDRA||||END| data/symbols/win32/mfc100u.exports||GHIDRA||||END| diff --git a/Ghidra/Features/Base/data/symbols/win16/commctrl.exports b/Ghidra/Features/Base/data/symbols/win16/commctrl.exports new file mode 100644 index 0000000000..cb2e72525c --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/commctrl.exports @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/commdlg.exports b/Ghidra/Features/Base/data/symbols/win16/commdlg.exports new file mode 100644 index 0000000000..fd8c7050b7 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/commdlg.exports @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/gdi.exports b/Ghidra/Features/Base/data/symbols/win16/gdi.exports new file mode 100644 index 0000000000..a7bdfb4dfa --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/gdi.exports @@ -0,0 +1,287 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/kernel.exports b/Ghidra/Features/Base/data/symbols/win16/kernel.exports new file mode 100644 index 0000000000..116c7691b5 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/kernel.exports @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/krnl386.exports b/Ghidra/Features/Base/data/symbols/win16/krnl386.exports new file mode 100644 index 0000000000..0aae843212 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/krnl386.exports @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/lzexpand.exports b/Ghidra/Features/Base/data/symbols/win16/lzexpand.exports new file mode 100644 index 0000000000..43ddef8429 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/lzexpand.exports @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/mmsystem.exports b/Ghidra/Features/Base/data/symbols/win16/mmsystem.exports new file mode 100644 index 0000000000..57e97f2c86 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/mmsystem.exports @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/toolhelp.exports b/Ghidra/Features/Base/data/symbols/win16/toolhelp.exports new file mode 100644 index 0000000000..b623fad135 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/toolhelp.exports @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/user.exports b/Ghidra/Features/Base/data/symbols/win16/user.exports new file mode 100644 index 0000000000..2252569b12 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/user.exports @@ -0,0 +1,417 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/ver.exports b/Ghidra/Features/Base/data/symbols/win16/ver.exports new file mode 100644 index 0000000000..c2fa210725 --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/ver.exports @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/Ghidra/Features/Base/data/symbols/win16/win87em.exports b/Ghidra/Features/Base/data/symbols/win16/win87em.exports new file mode 100644 index 0000000000..4085a7e12e --- /dev/null +++ b/Ghidra/Features/Base/data/symbols/win16/win87em.exports @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/Ghidra/Features/Base/src/main/help/help/topics/ImporterPlugin/importer.htm b/Ghidra/Features/Base/src/main/help/help/topics/ImporterPlugin/importer.htm index f06baf5dc6..534d1bca12 100644 --- a/Ghidra/Features/Base/src/main/help/help/topics/ImporterPlugin/importer.htm +++ b/Ghidra/Features/Base/src/main/help/help/topics/ImporterPlugin/importer.htm @@ -32,6 +32,8 @@
  • Common Object File Format (COFF)
  • Debug Symbols (DBG)
    • + +
  • DYLD Shared Cache
  • Executable and Linking Format (ELF)
    • @@ -259,7 +261,7 @@

    Common Options

    -

    These options appear many of the standard executable program formats such as ELF, PE, +

    These options appear in many of the standard executable program formats such as ELF, PE, etc.

    Apply Processor Defined Labels

    @@ -280,38 +282,32 @@ off, the symbols will move with the image base or the memory block.

    -

    Create Export Symbol Files

    +

    Load Local Libraries

    -

    Creates symbol files for each library used by an executable. The symbol file will - contain stack purge information for each exported symbol in the library. The symbol files - will be stored in <GHIDRA_INSTALL_DIR>/Ghidra/Features/Base/data/symbols/<OS>.

    - -

    When running Ghidra with .symbols files - created from an older operating system, you may receive the following warning - message:

    - -
    -

    Unable to locate [symbol_name] in - [<filepath>.exports]. - Please verify the version is correct.

    -
    - -

    This warning message indicates which symbols do not exist in the corresponding - .exports file. The only information lost by not including these symbols is - function purge and comments. If you require this information, manually delete the - .exports file and Ghidra will regenerate it.

    +

    Searches the executable's directory to recursively resolve the external libraries used + by the executable. The entire library dependency tree will be traversed in a depth-first + manner and a program will be created for each found library (if it doesn't exist already). + The external references + in these programs will be resolved.

    - -

    Load External Libraries

    + +

    Load System Libraries

    -

    Recursively resolves the external libraries used by the executable. The entire library - dependency tree will be traversed in a depth-first manner and a program will be created - for each library. The external references in - these program will be resolved.
    - The "..." button will bring up the Library Paths Dialog

    +

    Searches a user-defined path list to recursively resolve the external libraries used + by the executable. The entire library dependency tree will be traversed in a depth-first + manner and a program will be created for each found library (if it doesn't exist already). + The external references + in these program will be resolved.
    + The "Edit Paths" button will bring up the Library Paths Dialog

    +
    + +

    Recursive Library Load Depth

    + +
    +

    Specifies how many levels deep the depth-first library dependency tree will be + traversed when loading local or system libraries.

    @@ -352,13 +348,6 @@ the program is run. These sections will not be stored in a special address space called "other".

    - -

    Fixup Unresolved External Symbols

    - -
    -

    If selected, Ghidra will attemp to resolve external references against other programs - already imported into Ghidra that are in the destination folder for this import.

    -

    Intel Hex Options

    @@ -374,7 +363,7 @@

    Overlay

    -

    If selected, the bytes will be loaded as an initiailized overlay block. A new overlay space will be +

    If selected, the bytes will be loaded as an initialized overlay block. A new overlay space will be created with the same name as the Block Name.

    @@ -422,11 +411,58 @@

    The MZ format has only the Common Options.

    + +

    NE Options

    + +
    +

    The NE format has all the Common Options, plus:

    + +

    Perform Library Ordinal Lookup

    + +
    +

    Looks up and applies pre-generated exported symbol ordinal name mappings and stack + purge information. This information is stored in symbol files located in + <GHIDRA_INSTALL_DIR>/Ghidra/Features/Base/data/symbols/<OS>.

    + +

    If there is no pre-generated information for a given library but the ordinal name + mappings and/or stack purge information is extracted during the library load/analysis + process, the information will be cached locally to the user's .ghidra/ + directory to speed up future imports.

    +
    +

    PE Options

    The PE format has all the Common Options, plus:

    + +

    Perform Library Ordinal Lookup

    + +
    +

    Looks up and applies pre-generated exported symbol ordinal name mappings and stack + purge information. This information is stored in symbol files located in + <GHIDRA_INSTALL_DIR>/Ghidra/Features/Base/data/symbols/<OS>.

    + +

    If there is no pre-generated information for a given library but the ordinal name + mappings and/or stack purge information is extracted during the library load/analysis + process, the information will be cached locally to the user's .ghidra/ + directory to speed up future imports.

    + +

    When running Ghidra with symbol files + created from an older operating system, you may receive the following warning + message:

    + +
    +

    Unable to locate [symbol_name] in + [<filepath>.exports]. + Please verify the version is correct.

    +
    + +

    This warning message indicates which symbols do not exist in the corresponding + .exports file. The only information lost by not including these symbols is + function purge and comments. If you require this information, manually delete the + .exports file and Ghidra will regenerate it.

    +

    Parse CLI headers (if present)

    diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java index 380359d337..f375b942da 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java @@ -178,7 +178,7 @@ public class OptionsEditorPanel extends JPanel { public Component getEditorComponent(Option option) { //special case for load library paths - if (option.getName().equals(AbstractLibrarySupportLoader.LIB_OPTION_NAME)) { + if (option.getName().equals(AbstractLibrarySupportLoader.SYSTEM_LIBRARY_OPTION_NAME)) { return buildLoadLibraryPathsEditor(option); } @@ -224,11 +224,9 @@ public class OptionsEditorPanel extends JPanel { Boolean value = (Boolean) option.getValue(); boolean initialState = value != null ? value : false; GCheckBox jCheckBox = new GCheckBox("", initialState); - button.setEnabled(initialState); jCheckBox.addActionListener(e -> { boolean b = jCheckBox.isSelected(); option.setValue(b); - button.setEnabled(b); }); panel.add(jCheckBox, BorderLayout.WEST); panel.add(button, BorderLayout.EAST); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java index 7fd09a946c..aa48a41f2d 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java @@ -17,6 +17,7 @@ package ghidra.app.util.opinion; import java.io.File; import java.io.IOException; +import java.nio.file.AccessMode; import java.util.*; import java.util.stream.Collectors; @@ -24,34 +25,39 @@ import org.apache.commons.io.FilenameUtils; import ghidra.app.util.Option; import ghidra.app.util.bin.ByteProvider; -import ghidra.app.util.bin.RandomAccessByteProvider; +import ghidra.app.util.bin.FileByteProvider; import ghidra.app.util.importer.LibrarySearchPathManager; import ghidra.app.util.importer.MessageLog; import ghidra.formats.gfilesystem.FSRL; +import ghidra.formats.gfilesystem.FileSystemService; import ghidra.framework.model.*; -import ghidra.framework.options.Options; import ghidra.program.model.address.Address; import ghidra.program.model.lang.*; -import ghidra.program.model.listing.*; -import ghidra.program.model.symbol.*; +import ghidra.program.model.listing.Library; +import ghidra.program.model.listing.Program; +import ghidra.program.model.symbol.ExternalManager; import ghidra.util.Msg; -import ghidra.util.exception.*; +import ghidra.util.exception.CancelledException; +import ghidra.util.exception.InvalidInputException; import ghidra.util.task.TaskMonitor; import utilities.util.FileUtilities; /** * An abstract {@link Loader} that provides a framework to conveniently load {@link Program}s with * support for linking against libraries contained in other {@link Program}s. - * Subclasses are responsible for the actual load. *

    + * Subclasses may override various protected methods to customize how libraries are loaded. */ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader { - public static final String SYM_OPTION_NAME = "Create Export Symbol Files"; - public static final String LIB_OPTION_NAME = "Load External Libraries"; + public static final String LOCAL_LIBRARY_OPTION_NAME = "Load Local Libraries"; + static final boolean LOCAL_LIBRARY_OPTION_DEFAULT = false; - static final boolean IS_CREATE_EXPORT_SYMBOL_FILES_DEFAULT = true; - static final boolean IS_LOAD_LIBRARIES_DEFAULT = false; + public static final String SYSTEM_LIBRARY_OPTION_NAME = "Load System Libraries"; + static final boolean SYSTEM_LIBRARY_OPTION_DEFAULT = false; + + public static final String DEPTH_OPTION_NAME = "Recursive Library Load Depth"; + static final int DEPTH_OPTION_DEFAULT = 1; /** * Loads bytes in a particular format into the given {@link Program}. @@ -74,30 +80,23 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader DomainFolder programFolder, LoadSpec loadSpec, List

    @@ -204,149 +285,332 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader } /** - * Returns the path the loaded {@link ByteProvider} is located in. - *

    - * Special case when the ByteProvider specifies a {@link FSRL}, try to get the 'real' - * path on the local filesystem, otherwise return null. - * - * @param provider The {@link ByteProvider}. - * @return The path the loaded {@link ByteProvider} is located in. + * Creates a {@link ByteProvider} for the given library file + * + * @param libFile The library file to get a {@link ByteProvider} for + * @param loadSpec An optional {@link LoadSpec} the {@link ByteProvider} should conform to + * @param log The log + * @return A {@link ByteProvider} for the given library file, or null if one could not be + * created that matches the given {@link LoadSpec} + * @throws IOException If there was an IO-related issue */ - private String getProviderFilePath(ByteProvider provider) { - FSRL fsrl = provider.getFSRL(); - if ((fsrl != null) && !fsrl.getFS().hasContainer()) { - return FilenameUtils.getFullPathNoEndSeparator(fsrl.getPath()); - } - File f = provider.getFile(); - return (f != null) ? f.getParent() : null; + protected ByteProvider createLibraryByteProvider(File libFile, LoadSpec loadSpec, + MessageLog log) throws IOException { + return new FileByteProvider(libFile, FileSystemService.getInstance().getLocalFSRL(libFile), + AccessMode.READ); } /** - * Attempts to import all libraries listed in {@code unprocessedLibs}, placing the newly - * created {@link DomainObject} instances in {@code programList}. + * Checks whether or not the given library should be loaded. + *

    + * It may be appropriate to not load a specific library after examining its bytes. + * + * @param libraryName The name of the library + * @param libraryFile The library {@link File} + * @param provider The library bytes + * @param desiredLoadSpec The desired {@link LoadSpec} + * @param log The log + * @return True if the given library should be loaded; otherwise, false + * @throws IOException If an IO-related error occurred + */ + protected boolean shouldLoadLibrary(String libraryName, File libraryFile, ByteProvider provider, + LoadSpec desiredLoadSpec, MessageLog log) throws IOException { + if (matchSupportedLoadSpec(desiredLoadSpec, provider) == null) { + log.appendMsg("Skipping library which is the wrong architecture: " + libraryFile); + return false; + } + return true; + } + + /** + * Performs optional follow-on actions after an the given library has been loaded + * + * @param library The loaded library {@link Program} + * @param libraryName The name of the library + * @param libraryFile The library {@link File} + * @param provider The library bytes + * @param loadSpec The {@link LoadSpec} used for the load + * @param options The options + * @param log The log + * @param monitor A cancel.able monitor + * @return True if the library should be saved to the project; otherwise, false + * @throws IOException If an IO-related error occurred + * @throws CancelledException If the user cancelled the action + */ + protected boolean processLibrary(Program library, String libraryName, File libraryFile, + ByteProvider provider, LoadSpec loadSpec, List

    + * If the library path is a simple name without any path separators, only the given folder + * will be searched. + *

    + * If the library path has a path, it will be treated as a relative path under + * given folder and if found that {@link DomainFile} will be returned. + *

    + * If the library path has a path and it wasn't found under the given folder, the + * filename part of library path will be used to search the given folder for matches. + *

    + * @param libraryPath path with filename of the library to find + * @param folder {@link DomainFolder} within which imported libraries will be searched. + * If null this method will return null. + * @return The found {@link DomainFile} or null if not found + */ + private DomainFile findLibrary(String libraryPath, DomainFolder folder) { + if (folder == null) { + return null; + } - for (int i = 0; i < programs.size() && isCreateExportSymbolFiles(options); ++i) { - Program p = programs.get(i); + // Lookup by full project path + // NOTE: probably no need to support optional extensions and case-insensitivity for this case + String projectPath = appendPath(folder.getPathname(), libraryPath); + DomainFile ret = + folder.getProjectData().getFile(FilenameUtils.separatorsToUnix(projectPath)); + if (ret != null) { + return ret; + } - monitor.checkCanceled(); - monitor.setProgress(i); + // Quick lookup by library filename (ignoring full library path) in given folder. + // We try this first to hopefully avoid needing to iterate over the files in the folder + // factoring in case and extensions + String libraryName = FilenameUtils.getName(libraryPath); + if ((ret = folder.getFile(libraryName)) != null) { + return ret; + } - int id = p.startTransaction("apply"); - boolean success = false; - try { - applyLibrarySymbols(p, log, monitor); - applyImports(p, log, monitor); - - success = true; + // Factoring in case and optional file extensions, iterate over given folder looking for + // a match + boolean noExtension = FilenameUtils.getExtension(libraryName).equals(""); + Comparator comparator = getLibraryNameComparator(); + for (DomainFile file : folder.getFiles()) { + String candidateName = file.getName(); + if (isOptionalLibraryFilenameExtensions() && noExtension) { + candidateName = FilenameUtils.getBaseName(candidateName); } - finally { - p.endTransaction(id, success); + if (comparator.compare(candidateName, libraryName) == 0) { + return file; } } - LibraryLookupTable.cleanup(); + return null; } - protected boolean isCreateExportSymbolFiles(List

    + * Each search path directory will be searched for the library file in order. + *

    + * If the library file specifies a path, it is treated as a relative subdirectory of + * each search path directory that is searched, and if not found, the filename part of + * the library is used to search just the search path directory. + *

    + * If the library specifies an absolute path, its native path is searched on the local + * filesystem. + *

    + * @param libraryPath Either a path_and_filename, or just a filename of a library + * that should be searched for + * @param searchPaths A {@link List} of filesystem paths on the local filesystem that will be + * searched + * @return A {@link List} of files that match the requested library path + */ + private List findLibrary(String libraryPath, List searchPaths) { + + String libraryName = FilenameUtils.getName(libraryPath); + List results = new ArrayList<>(); + + for (String searchPath : searchPaths) { + + // ignore garbage entries: relative, non-existent, not directory + searchPath = FilenameUtils.normalizeNoEndSeparator(searchPath); + if (searchPath == null || searchPath.isEmpty()) { + continue; + } + File searchDir = new File(searchPath); + if (!searchDir.isAbsolute() || !searchDir.isDirectory()) { + continue; + } + + // 1) Try as possible subpath under the search path + String candidatePath = + FilenameUtils.separatorsToSystem(appendPath(searchPath, libraryPath)); + File f = resolveLibraryFile(new File(candidatePath)); + if (f == null || !f.isFile()) { + // 2) Fall back to looking for the library in the user specified search path, sans + // any subpath built into the library string + f = resolveLibraryFile(new File(searchDir, libraryName)); + } + if (f != null && f.isFile() && !results.contains(f)) { + results.add(f); } } - return isCreateExportSymbolFiles; - } - protected boolean isLoadLibraries(List

    - * Other programs in the programs list are matched first, then the - * ghidraLibSearchFolders are searched for matches. + * Other programs in the given list are matched first, then the ghidraLibSearchFolders are + * searched for matches. * * @param programs the list of programs to resolve against each other. Programs not saved - * to the project will be considered as a valid external library. + * to the project will be considered as a valid external library. * @param domainFolder the {@link DomainFolder} folder within which imported libraries will - * be searched. This folder will be searched if a library is not found within the - * list of programs supplied. If null, only the list of programs will be considered. + * be searched. This folder will be searched if a library is not found within the + * list of programs supplied. If null, only the list of programs will be considered. * @param saveIfModified flag to have this method save any programs it modifies * @param messageLog log for messages. * @param monitor the task monitor * @throws IOException if there was an IO-related problem resolving. * @throws CancelledException if the user cancelled the load. */ - private void resolveExternalLibs(List programs, DomainFolder domainFolder, + private void fixupExternalLibraries(List programs, DomainFolder domainFolder, boolean saveIfModified, MessageLog messageLog, TaskMonitor monitor) throws CancelledException, IOException { @@ -439,9 +701,9 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader } monitor.setMessage("Resolving..." + program.getName()); - int id = program.startTransaction("resolving external references"); + int id = program.startTransaction("Resolving external references"); try { - resolveExternalLibs(program, progsByName, domainFolder, monitor, messageLog); + resolveExternalLibraries(program, progsByName, domainFolder, monitor, messageLog); } finally { program.endTransaction(id, true); @@ -453,29 +715,29 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader } /** - * Fix up program's external Library entries so - * that they point to a path in the ghidra project. + * Fix up program's external library entries so that they point to a path in the project. *

    - * Other programs in the progsByName map are matched first, then the - * ghidraLibSearchFolders are searched for matches. + * Other programs in the map are matched first, then the ghidraLibSearchFolders + * are searched for matches. * * @param program the program whose Library entries are to be resolved. An open transaction - * on program is required. + * on program is required. * @param progsByName map of recently imported programs to be considered - * first when resolving external Libraries. Programs not saved to the project - * will be ignored. + * first when resolving external Libraries. Programs not saved to the project + * will be ignored. * @param domainFolder the {@link DomainFolder} folder within which imported libraries will - * be searched. This folder will be searched if a library is not found within the - * progsByName map. If null, only progsByName will be considered. + * be searched. This folder will be searched if a library is not found within the + * progsByName map. If null, only progsByName will be considered. * @param messageLog log for messages. * @param monitor the task monitor * @throws CancelledException if the user cancelled the load. */ - private void resolveExternalLibs(Program program, Map progsByName, + private void resolveExternalLibraries(Program program, Map progsByName, DomainFolder domainFolder, TaskMonitor monitor, MessageLog messageLog) throws CancelledException { ExternalManager extManager = program.getExternalManager(); String[] extLibNames = extManager.getExternalLibraryNames(); + messageLog.appendMsg("Linking external programs to " + program.getName() + "..."); for (String externalLibName : extLibNames) { if (Library.UNKNOWN.equals(externalLibName)) { continue; @@ -493,17 +755,15 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader } else { DomainFile alreadyImportedLib = - findAlreadyImportedLibrary(externalLibName, domainFolder); + findLibrary(externalLibName, domainFolder); if (alreadyImportedLib != null) { extManager.setExternalPath(externalLibName, alreadyImportedLib.getPathname(), false); - messageLog.appendMsg( - " [" + program.getName() + ":" + externalLibName + "] -> [" + - alreadyImportedLib.getPathname() + "] (previously imported)"); + messageLog.appendMsg(" [" + externalLibName + "] -> [" + + alreadyImportedLib.getPathname() + "] (previously imported)"); } else { - messageLog.appendMsg( - " [" + program.getName() + ":" + externalLibName + "] -> not found"); + messageLog.appendMsg(" [" + externalLibName + "] -> not found"); } } } @@ -513,534 +773,142 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader } } - protected Program findLibraryWithCaseCorrectSearch(Map progsByName, - String libName) { - Comparator comparator = getLibNameComparator(); - for (String s : progsByName.keySet()) { - if (comparator.compare(libName, s) == 0) { - return progsByName.get(s); + /** + * A library that has not been processed by the loader yet + * + * @param name The name of the library + * @param depth The recursive load depth of the library (based on the original binary being + * loaded) + */ + private record UnprocessedLibrary(String name, int depth) {/**/} + + /** + * Creates a new {@link Queue} of {@link UnprocessedLibrary}s, initialized filled with the + * library names in the given list + * + * @param libraryNames A {@link List} of unprocessed library names + * @param depth The initial load depth of each library + * @return A {@link Queue} of {@link UnprocessedLibrary}s + */ + private Queue createUnprocessedQueue(List libraryNames, int depth) { + return libraryNames.stream() + .map(name -> new UnprocessedLibrary(name, depth)) + .collect(Collectors.toCollection(LinkedList::new)); + } + + /** + * Gets a {@link List} of priority-ordered paths used to search for libraries + * + * @param provider The {@link ByteProvider} of the program being loaded + * @param options The options + * @return A {@link List} of priority-ordered paths used to search for libraries + */ + private List getLibrarySearchPaths(ByteProvider provider, List

    - * If libPathFilename is a simple name without any path separators, only the - * importFolder folder will be searched. - *

    - * If libPathFilename has a path, it will be treated as a relative path under - * {@code importFolder} and if found that DomainFile will be returned. - *

    - * If libPathFilename has a path and it wasn't found under importFolder, the - * filename part of libPathFilename will be used to search the importFolder for matches. - *

    - * @param libPathFilename String path with filename of the library to find - * @param domainFolder {@link DomainFolder} folder within which imported libraries will - * be searched. If null this method will return null. - * @return found {@link DomainFile} or null if not found + * Appends the given path elements to form a single path + * + * @param pathElements The path elements to append to one another + * @return A single path consisting of the given path elements appended together */ - protected DomainFile findAlreadyImportedLibrary(String libPathFilename, - DomainFolder domainFolder) { - if (domainFolder == null) { - return null; - } - - // Lookup by full project path - // NOTE: probably no need to support optional extensions and case-insensitivity for this case - String projectPath = appendPath(domainFolder.getPathname(), libPathFilename); - DomainFile ret = - domainFolder.getProjectData().getFile(FilenameUtils.separatorsToUnix(projectPath)); - if (ret != null) { - return ret; - } - - // Quick lookup by library filename (ignoring full library path) in given folder. - // We try this first to hopefully avoid needing to iterate over the files in the folder - // factoring in case and extensions - String libName = FilenameUtils.getName(libPathFilename); - if ((ret = domainFolder.getFile(libName)) != null) { - return ret; - } - - // Factoring in case and optional file extensions, iterate over given folder looking for - // a match - boolean noExtension = FilenameUtils.getExtension(libName).equals(""); - Comparator libNameComparator = getLibNameComparator(); - for (DomainFile file : domainFolder.getFiles()) { - String candidateName = file.getName(); - if (isOptionalLibraryFilenameExtensions() && noExtension) { - candidateName = FilenameUtils.getBaseName(candidateName); - } - if (libNameComparator.compare(candidateName, libName) == 0) { - return file; - } - } - - return null; - } - - private static String appendPath(String... pathStrs) { + private String appendPath(String... pathElements) { StringBuilder sb = new StringBuilder(); - for (String pathEle : pathStrs) { - if (pathEle == null || pathEle.isEmpty()) { + for (String pathElement : pathElements) { + if (pathElement == null || pathElement.isEmpty()) { continue; } boolean sbEndsWithSlash = sb.length() > 0 && "/\\".indexOf(sb.charAt(sb.length() - 1)) != -1; - boolean eleStartsWithSlash = "/\\".indexOf(pathEle.charAt(0)) != -1; + boolean elementStartsWithSlash = "/\\".indexOf(pathElement.charAt(0)) != -1; - if (!sbEndsWithSlash && !eleStartsWithSlash && sb.length() > 0) { + if (!sbEndsWithSlash && !elementStartsWithSlash && sb.length() > 0) { sb.append("/"); } - else if (eleStartsWithSlash && sbEndsWithSlash) { - pathEle = pathEle.substring(1); + else if (elementStartsWithSlash && sbEndsWithSlash) { + pathElement = pathElement.substring(1); } - sb.append(pathEle); + sb.append(pathElement); } return sb.toString(); } /** - * Searches the local filesystem for the specified library file, returning a List - * of possible candidate files. - *

    - * Each importPath directory will be searched for the library file in order. - *

    - * If the library file specifies a path, it is treated as a relative subdirectory of - * each importPath directory that is searched, and if not found, the filename part of - * the library is used to search just the importPath directory. - *

    - * If the library specifies a path, its native path is searched on the local filesystem. - *

    - * @param libPathFilename - either a path_and_filename, or just a filename of a library - * that should be searched for. - * @param importPaths - list of filesystem paths on the local computer that will be - * searched. - * @return a List of Files (possibly empty, never null) that match the requested filename. - */ - private List findLibraryFileToImport(String libPathFilename, List importPaths) { - - String libName = FilenameUtils.getName(libPathFilename); - List results = new ArrayList<>(); - - for (String importPath : importPaths) { - - // ignore garbage importPath entries: relative, non-existent, not directory - importPath = FilenameUtils.normalizeNoEndSeparator(importPath); - if (importPath == null || importPath.isEmpty()) { - continue; - } - File importPathDir = new File(importPath); - if (!importPathDir.isAbsolute() || !importPathDir.isDirectory()) { - continue; - } - - // 1) Try as possible subpath under the importPath - String candidatePath = - FilenameUtils.separatorsToSystem(appendPath(importPath, libPathFilename)); - File f = resolveLibraryFile(new File(candidatePath)); - if (f == null || !f.isFile()) { - // 2) Fall back to looking for the library in the user specified importPath, sans any - // subpath built into the library string. - f = resolveLibraryFile(new File(importPathDir, libName)); - } - if (f != null && f.isFile() && !results.contains(f)) { - results.add(f); - } - } - - boolean searchLocalFileSystemAlso = true; - boolean libSpecifiesPath = FilenameUtils.getPrefixLength(libPathFilename) > 0; - if (searchLocalFileSystemAlso && libSpecifiesPath) { - // 3) Search the local filesystem (as if the importPath list contained "/") - // if the specified library string specifies a path. - File f = resolveLibraryFile(new File(libPathFilename)); - if (f != null && f.isAbsolute() && f.isFile() && !results.contains(f)) { - results.add(f); - } - } - - return results; - } - - /** - * Imports a library file into a ghidra project. + * Ensures the given {@link LoadSpec} matches one supported by the loader * - * @param libName the name of the library to import - * @param libFolder the library folder - * @param libFile the library file to load - * @param loadSpec the {@link LoadSpec} - * @param options the load options - * @param log the message log - * @param consumer consumer object for the {@link Program} generated - * @param unprocessedLibs queue of libraries that need to be loaded - * @param programList list of programs to add the imported library to - * @param monitor the task monitor - * @return true if the load was successful - * @throws CancelledException if the user cancelled the load operation - * @throws IOException if there was an error during the load + * @param desiredLoadSpec The desired {@link LoadSpec} + * @param provider The provider + * @return A supported {@link LoadSpec} that matches the desired one, or null of none matched + * @throws IOException if there was an IO-related error */ - protected boolean importLibrary(String libName, DomainFolder libFolder, File libFile, - LoadSpec loadSpec, List

    + * Special case when the ByteProvider specifies a {@link FSRL}, try to get the 'real' + * path on the local filesystem, otherwise return null. + * + * @param provider The {@link ByteProvider}. + * @return The path the loaded {@link ByteProvider} is located in. + */ + private String getProviderFilePath(ByteProvider provider) { + FSRL fsrl = provider.getFSRL(); + if ((fsrl != null) && !fsrl.getFS().hasContainer()) { + return FilenameUtils.getFullPathNoEndSeparator(fsrl.getPath()); + } + File f = provider.getFile(); + return (f != null) ? f.getParent() : null; + } + + /** + * Gets a {@link Comparator} for comparing library filenames + * + * @return A {@link Comparator} for comparing library filenames + */ + private Comparator getLibraryNameComparator() { + return isCaseInsensitiveLibraryFilenames() + ? String.CASE_INSENSITIVE_ORDER + : (s1, s2) -> s1.compareTo(s2); + } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java new file mode 100644 index 0000000000..c2b9a11241 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java @@ -0,0 +1,382 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.util.opinion; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import ghidra.app.util.Option; +import ghidra.app.util.bin.ByteProvider; +import ghidra.app.util.importer.MessageLog; +import ghidra.framework.model.DomainFolder; +import ghidra.framework.model.DomainObject; +import ghidra.framework.options.Options; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.Program; +import ghidra.program.model.symbol.*; +import ghidra.util.Msg; +import ghidra.util.exception.*; +import ghidra.util.task.TaskMonitor; + +/** + * An abstract {@link Loader} that provides support for programs that link to external libraries + * with an ordinal mechanism. Supports caching library lookup information to XML files. + */ +public abstract class AbstractOrdinalSupportLoader extends AbstractLibrarySupportLoader { + + public static final String ORDINAL_LOOKUP_OPTION_NAME = "Perform Library Ordinal Lookup"; + static final boolean ORDINAL_LOOKUP_OPTION_DEFAULT = true; + + @Override + public List

  • -loader-applyLabels <true|false>
  • -loader-anchorLabels <true|false>
    • -
  • -loader-createExportSymbolFiles <true|false>
    • -
  • -loader-loadExternalLibs <true|false>
    • +
  • -loader-loadLocalLibraries <true|false>
    • +
  • -loader-loadSystemLibraries <true|false>
    • +
  • -loader-libraryLoadDepth <depth>
  • -loader-applyRelocations <true|false>
  • -loader-imagebase <imagebase3>
  • -loader-dataImageBase <dataImageBase4>
  • -loader-includeOtherBlocks <true|false>
    • -
  • -loader-resolveExternalSymbols <true|false>
  • -loader PeLoader
  • -loader MachoLoader