Merge remote-tracking branch 'origin/GP-298_FidNamespace' into Ghidra_9.2

Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java

@@ -405,11 +405,6 @@ public class FidStatistics extends GhidraScript {
 			}
 			String funcName = chooseFunctionName(result);
 			NameVersions nameVersions = NameVersions.generate(funcName, program);
-			String strippedTemplateName = null;
-			if (nameVersions.demangledBaseName != null) {
-				strippedTemplateName =
-					MatchNameAnalysis.removeTemplateParams(nameVersions.demangledBaseName);
-			}
 			boolean exactNameMatch = false;
 			Iterator<String> iter = matchAnalysis.getRawNameIterator();
 			while(iter.hasNext()) {
@@ -450,14 +445,12 @@ public class FidStatistics extends GhidraScript {
 					exactNameMatch = true;
 					break;
 				}
-				if (matchNames.demangledBaseName != null && strippedTemplateName != null) {
-					String strippedName =
-						MatchNameAnalysis.removeTemplateParams(matchNames.demangledBaseName);
-					if (strippedName != null) {
-						if (checkNames(strippedName, strippedTemplateName)) {
-							exactNameMatch = true;
-							break;
-						}
+				if (nameVersions.demangledNoTemplate != null &&
+					matchNames.demangledNoTemplate != null) {
+					if (checkNames(nameVersions.demangledNoTemplate,
+						matchNames.demangledNoTemplate)) {
+						exactNameMatch = true;
+						break;
 					}
 				}
 			}

Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java

@@ -41,7 +41,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 
 	private MatchNameAnalysis nameAnalysis = new MatchNameAnalysis();
 	private AddressSet affectedLocations = new AddressSet();
-	private TreeMap<String, Address> multiMatchNames = new TreeMap<String, Address>();
+	private TreeMap<String, List<Address>> multiMatchNames = new TreeMap<String, List<Address>>();
 	private LinkedList<Address> conflictFunctions = new LinkedList<Address>();
 	private boolean alwaysApplyFidLabels;
 	private float scoreThreshold;
@@ -128,18 +128,16 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 		nameAnalysis.analyzeLibraries(result.matches, MAGIC_MULTIPLE_LIBRARY_LIMIT, monitor);
 
 		String newFunctionName = null;
-		if (nameAnalysis.numNames() == 1) {
-			newFunctionName = nameAnalysis.getNameIterator().next();
-		}
-
-		if (nameAnalysis.numSimilarNames() == 1) { // If all names are the same, up to a difference in '_' prefix
+		if (nameAnalysis.numNames() == 1) { // If all names are the same, up to a difference in '_' prefix
 			bookmarkContents = "Library Function - Single Match, ";
 			plateCommentContents = "Library Function - Single Match";
+			newFunctionName = nameAnalysis.getNameIterator().next();
 		}
 		else { // If names are different in some way
 			bookmarkContents = "Library Function - Multiple Matches, ";
 			plateCommentContents = "Library Function - Multiple Matches";
-			if (nameAnalysis.numNames() == 1) {
+			if (nameAnalysis.getMostOptimisticCount() == 1) {
+				newFunctionName = nameAnalysis.getMostOptimisticName();
 				plateCommentContents = plateCommentContents + " With Same Base Name";
 				bookmarkContents = bookmarkContents + "Same ";
 			}
@@ -157,16 +155,26 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 			monitor);
 	}
 
+	/**
+	 * Construct list of names as they should appear in the comment for the function.
+	 * @param monitor is the task monitor
+	 * @return the list of names as a formatted String
+	 * @throws CancelledException if the user cancels the task
+	 */
 	private String listNames(TaskMonitor monitor) throws CancelledException {
 		StringBuilder buffer = new StringBuilder();
 
 		int counter = 0;
-
 		Iterator<String> iterator = nameAnalysis.getNameIterator();
 		while (iterator.hasNext()) {
 			monitor.checkCanceled();
+			String display = iterator.next();
+			NameVersions versions = nameAnalysis.getVersions(display);
+			if (versions != null && versions.demangledFull != null) {
+				display = versions.demangledFull;
+			}
 			buffer.append(' ');
-			buffer.append(iterator.next());
+			buffer.append(display);
 			buffer.append('\n');
 			counter++;
 			if (counter > 3) {
@@ -332,32 +340,30 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 	// Also checks those locations to see if there is only one label left and if so, removes the "FID conflict" bookmark.
 	private void removeConflictSymbols(Function function, String matchName, TaskMonitor monitor) {
 
-		Address addr = multiMatchNames.get(matchName);
-		if (addr == null) {
+		List<Address> list = multiMatchNames.get(matchName);
+		if (list == null) {
 			return;
 		}
 		Program program = function.getProgram();
-		int numSymbols = deleteSymbol(matchName, addr, program);
-		if (numSymbols <= 1) {
-			// Only one symbol left, delete the "FID conflict" bookmark
-			BookmarkManager bookmarkManager = program.getBookmarkManager();
-			Bookmark bookmark = bookmarkManager.getBookmark(addr, BookmarkType.ANALYSIS,
-				FIDCONFLICT_BOOKMARK_CATEGORY);
-			if (bookmark != null) {
-				bookmarkManager.removeBookmark(bookmark);
+		for (Address addr : list) {
+			int numSymbols = deleteSymbol(matchName, addr, program);
+			if (numSymbols <= 1) {
+				// Only one symbol left, delete the "FID conflict" bookmark
+				BookmarkManager bookmarkManager = program.getBookmarkManager();
+				Bookmark bookmark = bookmarkManager.getBookmark(addr, BookmarkType.ANALYSIS,
+					FIDCONFLICT_BOOKMARK_CATEGORY);
+				if (bookmark != null) {
+					bookmarkManager.removeBookmark(bookmark);
+				}
 			}
 		}
 	}
 
-	private int addFunctionLabelMultipleMatches(Function function, TaskMonitor monitor)
+	private void addFunctionLabelMultipleMatches(Function function, TaskMonitor monitor)
 			throws CancelledException {
 
 		Program program = function.getProgram();
 
-		if (nameAnalysis.numNames() >= MAGIC_MULTIPLE_MATCH_LIMIT) {
-			return nameAnalysis.numNames();
-		}
-
 		Symbol symbol = function.getSymbol();
 		boolean preexistingSymbol = (symbol != null && symbol.getSource() != SourceType.DEFAULT);
 
@@ -368,7 +374,12 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 		for (String functionName : unusedNames) {
 			monitor.checkCanceled();
 			addSymbolToFunction(function, functionName);
-			multiMatchNames.put(functionName, addr);
+			List<Address> list = multiMatchNames.get(functionName);
+			if (list == null) {
+				list = new LinkedList<Address>();
+				multiMatchNames.put(functionName, list);
+			}
+			list.add(addr);
 		}
 
 		if (unusedNames.size() > 1) {
@@ -382,7 +393,6 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 				"Multiple likely matching functions");
 			}
 		}
-		return unusedNames.size();
 	}
 
 	/**
@@ -432,7 +442,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 	 * Takes a set of FID matching names and returns a subset that includes only names that don't exist
 	 * somewhere else in the program.
 	 */
-	private static Set<String> getFIDNamesThatDontExistSomewhereElse(Program program,
+	private Set<String> getFIDNamesThatDontExistSomewhereElse(Program program,
 			Iterator<String> iter) {
 
 		Set<String> unusedNames = new HashSet<String>();
@@ -441,27 +451,42 @@ public class ApplyFidEntriesCommand extends BackgroundCommand {
 			String name = iter.next();
 			if (!nameExistsSomewhereElse(symbolTable, name)) {
 				unusedNames.add(name);
+				if (unusedNames.size() > MAGIC_MULTIPLE_MATCH_LIMIT) {
+					break;
+				}
 			}
 		}
 		return unusedNames;
 	}
 
+	private static boolean containsPrimarySymbol(SymbolTable symTab, String name) {
+		List<Symbol> syms = symTab.getSymbols(name, null);
+		for (Symbol symbol : syms) {
+			SymbolType type = symbol.getSymbolType();
+			if (type != SymbolType.FUNCTION && type != SymbolType.LABEL) {
+				continue;
+			}
+			if (symbol.isPrimary()) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 	//Check to see if other functions exist with the same baseName or _baseName or __baseName
-	private static boolean nameExistsSomewhereElse(SymbolTable symTab, String baseName) {
-
+	private boolean nameExistsSomewhereElse(SymbolTable symTab, String baseName) {
+		if (multiMatchNames.containsKey(baseName)) {
+			// If this name is part of a multimatch, don't treat as a definitive label
+			return false;
+		}
 		//I did it this way because doing it with an iterator and wildcard was really really slow
-		List<Symbol> globalSymbols = symTab.getLabelOrFunctionSymbols(baseName, null);
-		if (!globalSymbols.isEmpty()) {
+		if (containsPrimarySymbol(symTab, baseName)) {
 			return true;
 		}
-
-		globalSymbols = symTab.getLabelOrFunctionSymbols("_" + baseName, null);
-		if (!globalSymbols.isEmpty()) {
+		if (containsPrimarySymbol(symTab, "_" + baseName)) {
 			return true;
 		}
-
-		globalSymbols = symTab.getLabelOrFunctionSymbols("__" + baseName, null);
-		if (!globalSymbols.isEmpty()) {
+		if (containsPrimarySymbol(symTab, "__" + baseName)) {
 			return true;
 		}
 

Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/MatchNameAnalysis.java

@@ -30,58 +30,131 @@ import ghidra.util.task.TaskMonitor;
  */
 public class MatchNameAnalysis {
 	private Set<String> finalNameList = null;
+	private TreeMap<String, NameVersions> versionMap = null;
 	private TreeSet<String> rawNames = null;
 	private TreeSet<String> similarBaseNames = null;
+	private TreeSet<String> demangledNameNoTemplate = null;
 	private TreeSet<String> exactDemangledBaseNames = null;
 	private TreeSet<String> libraries = null;
 	
-	private int mostOptimisticCount;			// What is most optimistic (smallest) number of matches
-												// Once duplicates and similar base names are taken into account
 	private float overallScore = 0.0f;
 
+	/**
+	 * @return the number of deduped symbol names
+	 */
 	public int numNames() {
 		return finalNameList.size();
 	}
 
+	/**
+	 * @return an iterator to the deduped list of raw names (similar names are not collapsed)
+	 */
 	public Iterator<String> getRawNameIterator() {
 		return rawNames.iterator();
 	}
 
+	/**
+	 * Check if the given name is contained in the list of matches
+	 * @param name is the given name
+	 * @return true if the name is in the list
+	 */
 	public boolean containsRawName(String name) {
 		return rawNames.contains(name);
 	}
 
+	/**
+	 * @return an iterator to the final list of deduped symbol names
+	 */
 	public Iterator<String> getNameIterator() {
 		return finalNameList.iterator();
 	}
 
-	public int numSimilarNames() {
-		return similarBaseNames.size();
-	}
-
+	/**
+	 * @return the number of symbols in the deduped list of matching libraries
+	 */
 	public int numLibraries() {
 		return libraries.size();
 	}
 
+	/**
+	 * @return an iterator to the deduped list of matching libraries
+	 */
 	public Iterator<String> getLibraryIterator() {
 		return libraries.iterator();
 	}
 
+	/**
+	 * Get an object with all the given versions of the given raw name
+	 * @param raw is the raw name
+	 * @return the corresponding NameVersions object
+	 */
+	public NameVersions getVersions(String raw) {
+		return versionMap.get(raw);
+	}
+
+	/**
+	 * Run through all deduping strategies and return the number of unique symbols given
+	 * by the best strategy.
+	 * @return number of unique symbols given by the optimal deduping strategy
+	 */
 	public int getMostOptimisticCount() {
-		return mostOptimisticCount;
+		int count = rawNames.size();
+		if (similarBaseNames.size() < count) {
+			count = similarBaseNames.size();
+		}
+		if (demangledNameNoTemplate != null && demangledNameNoTemplate.size() < count) {
+			count = demangledNameNoTemplate.size();
+		}
+		if (exactDemangledBaseNames != null && exactDemangledBaseNames.size() < count) {
+			count = exactDemangledBaseNames.size();
+		}
+		return count;
+	}
+
+	/**
+	 * Run through ALL deduping strategies and if one results in a single label, return that label.
+	 * Otherwise return null.
+	 * @return a unique name describing all matches or null
+	 */
+	public String getMostOptimisticName() {
+		if (rawNames.size() == 1) {
+			return rawNames.first();
+		}
+		if (similarBaseNames.size() == 1) {
+			return similarBaseNames.first();
+		}
+		if (demangledNameNoTemplate != null && demangledNameNoTemplate.size() == 1) {
+			return demangledNameNoTemplate.first();
+		}
+		if (exactDemangledBaseNames != null && exactDemangledBaseNames.size() == 1) {
+			return exactDemangledBaseNames.first();
+		}
+		return null;
 	}
 
 	public float getOverallScore() {
 		return overallScore;
 	}
 
+	/**
+	 * Analyze a list of FID matches from a single address, deciding on the final list of symbols
+	 * that will be associated with the address by deduping and omitting similar names.
+	 * The final list is in finalNameList.
+	 * Demangling and template stripping can produce an even shorter list than finalNameList.
+	 * This is optionally available through getMostOptimisticName().
+	 * @param matches is the set of FID matches to analyze
+	 * @param program is the Program
+	 * @param monitor is the monitor
+	 * @throws CancelledException if the user cancels the task
+	 */
 	public void analyzeNames(List<FidMatch> matches, Program program, TaskMonitor monitor)
 				throws CancelledException {
 
+		versionMap = new TreeMap<String, NameVersions>();
 		rawNames = new TreeSet<String>();
 		similarBaseNames = new TreeSet<String>();
+		demangledNameNoTemplate = new TreeSet<String>();
 		exactDemangledBaseNames = new TreeSet<String>();
-		int cannotDemangle = 0;
 
 		for (FidMatch match : matches) {
 			monitor.checkCanceled();
@@ -91,36 +164,27 @@ public class MatchNameAnalysis {
 			NameVersions nameVersions = NameVersions.generate(function.getName(), program);
 			// Put exact base names in a HashSet
 			if (nameVersions.rawName != null) {
+				versionMap.put(nameVersions.rawName, nameVersions);
 				rawNames.add(nameVersions.rawName);				// Dedup the raw names
 				similarBaseNames.add(nameVersions.similarName);	// Dedup names with underscores removed
-				if (nameVersions.demangledBaseName != null) {
+				if (nameVersions.demangledNoTemplate != null && demangledNameNoTemplate != null) {
+					demangledNameNoTemplate.add(nameVersions.demangledNoTemplate);
+				}
+				else {
+					demangledNameNoTemplate = null;		// Get rid of container if we can't strip everything
+				}
+				if (nameVersions.demangledBaseName != null && exactDemangledBaseNames != null) {
 					exactDemangledBaseNames.add(nameVersions.demangledBaseName);		// Dedup demangled base name
 				}
 				else {
-					cannotDemangle += 1;
+					exactDemangledBaseNames = null;		// Get rid of container if we can't demangle everything
 				}
 			}
 		}
 
-		String singleName = null;
-		mostOptimisticCount = rawNames.size();
 		finalNameList = rawNames;
-		if (rawNames.size() == 1) {
-			singleName = rawNames.first();
-		}
-		else {
-			singleName = findCommonBaseName();
-			mostOptimisticCount = similarBaseNames.size();
-			if (singleName == null) {
-				singleName = findCommonDemangledBaseName(cannotDemangle);
-				if (exactDemangledBaseNames.size() > 0 &&
-					exactDemangledBaseNames.size() < mostOptimisticCount) {
-					mostOptimisticCount = exactDemangledBaseNames.size();
-				}
-			}
-		}
+		String singleName = findCommonBaseName();
 		if (singleName != null) {
-			mostOptimisticCount = 1;
 			finalNameList = Collections.singleton(singleName);
 		}
 		else if (rawNames.size() > similarBaseNames.size()) {
@@ -134,6 +198,15 @@ public class MatchNameAnalysis {
 		}
 	}
 	
+	/**
+	 * Collect strings describing the library descriptor of a set of FID matches.
+	 * Dedup the list trying to get the size down below a specified limit, stripping
+	 * version and family information from the library string if necessary.
+	 * @param matches is the set of FID matches
+	 * @param libraryLimit is the specified size limit
+	 * @param monitor is a task monitor
+	 * @throws CancelledException if the user cancels the task
+	 */
 	public void analyzeLibraries(Collection<FidMatch> matches,int libraryLimit,TaskMonitor monitor)
 			throws CancelledException {
 		libraries = new TreeSet<String>();
@@ -176,65 +249,15 @@ public class MatchNameAnalysis {
 		}
 	}
 
+	/**
+	 * Make a final decision based on the deduping strategies if there is a single
+	 * matching name that describes all matches.
+	 * @return the single matching name or null
+	 */
 	private String findCommonBaseName() {
-		if (similarBaseNames.size() == 1) {
-			return rawNames.iterator().next();
+		if (rawNames.size() == 1 || similarBaseNames.size() == 1) {
+			return rawNames.first();
 		}
 		return null;
 	}
-
-	/**
-	 * If there exists an initial set of template parameters bracketed by '<' and '>'
-	 * in this name, strip them from the name.
-	 * @param name is the function name to strip
-	 * @return the stripped name or null if no parameters present
-	 */
-	public static String removeTemplateParams(String name) {
-		int pos1 = name.indexOf('<');
-		if (pos1 < 0) {
-			return null;
-		}
-		int nesting = 1;
-		int pos2;
-		for (pos2 = pos1 + 1; pos2 < name.length(); ++pos2) {
-			char c = name.charAt(pos2);
-			if (c == '<') {
-				nesting += 1;
-			}
-			else if (c == '>') {
-				nesting -= 1;
-				if (nesting == 0) {
-					break;
-				}
-			}
-		}
-		if (nesting != 0) {
-			return null;
-		}
-		return name.substring(0, pos1 + 1) + name.substring(pos2);
-	}
-
-	private String findCommonDemangledBaseName(int cannotDemangle) {
-		if (cannotDemangle > 0) {
-			return null;			// Couldn't demangle everything, so no way we can have a common base
-		}
-		if (exactDemangledBaseNames.size() == 1) {
-			return exactDemangledBaseNames.iterator().next();
-		}
-		// If we don't have a unique demangled name, try excising template parameters
-		String finalName = null;
-		for (String name : exactDemangledBaseNames) {
-			String templateFree = removeTemplateParams(name);
-			if (templateFree == null) {
-				return null;		// At least one name has no template parameters
-			}
-			if (finalName == null) {
-				finalName = templateFree;
-			}
-			else if (!finalName.equals(templateFree)) {
-				return null;
-			}
-		}
-		return finalName;
-	}
 }

Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/NameVersions.java

@@ -21,11 +21,14 @@ import ghidra.program.model.listing.Program;
 public class NameVersions {
 	public String rawName;			// Original name
 	public String similarName;		// Name with underscores removed
+	public String demangledNoTemplate;	// Demangled string with (first) template removed
 	public String demangledBaseName;	// Base name of the demangled string
+	public String demangledFull;	// The full demangled signature
 	
 	public NameVersions(String raw) {
 		rawName = raw;
 		similarName = null;
+		demangledNoTemplate = null;
 		demangledBaseName = null;
 	}
 
@@ -55,6 +58,38 @@ public class NameVersions {
 		return name;
 	}
 
+	/**
+	 * If there exists an initial set of template parameters bracketed by '<' and '>'
+	 * in the given demangled name, strip them.
+	 * @param demangledObj is the object holding the demangled name
+	 * @return the stripped name or null if no parameters present
+	 */
+	private static String removeTemplateParams(DemangledObject demangledObj) {
+		String name = demangledObj.getDemangledName();
+		int pos1 = name.indexOf('<');
+		if (pos1 < 0) {
+			return null;
+		}
+		int nesting = 1;
+		int pos2;
+		for (pos2 = pos1 + 1; pos2 < name.length(); ++pos2) {
+			char c = name.charAt(pos2);
+			if (c == '<') {
+				nesting += 1;
+			}
+			else if (c == '>') {
+				nesting -= 1;
+				if (nesting == 0) {
+					break;
+				}
+			}
+		}
+		if (nesting != 0) {
+			return null;
+		}
+		return name.substring(0, pos1 + 1) + name.substring(pos2);
+	}
+
 	private static String constructBaseName(DemangledObject demangledObj) {
 		String origName = demangledObj.getName();
 		String name = origName.replaceFirst("_*", "");
@@ -87,6 +122,8 @@ public class NameVersions {
 		if (rawName != null) {
 			DemangledObject demangledObj = demangle(program, rawName);
 			if (demangledObj != null) {
+				result.demangledFull = demangledObj.getOriginalDemangled();
+				result.demangledNoTemplate = removeTemplateParams(demangledObj);
 				result.demangledBaseName = constructBaseName(demangledObj);
 			}