Merge remote-tracking branch

'origin/GP-1171_dev747368_DWARF_MIPSPro_64bit' (Closes #3223)

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DWARFCompilationUnit.java

@@ -19,6 +19,7 @@ import java.io.IOException;
 import java.util.*;
 
 import ghidra.app.util.bin.BinaryReader;
+import ghidra.app.util.bin.format.dwarf4.DWARFUtil.LengthResult;
 import ghidra.app.util.bin.format.dwarf4.next.DWARFProgram;
 import ghidra.util.Msg;
 import ghidra.util.exception.CancelledException;
@@ -126,32 +127,12 @@ public class DWARFCompilationUnit {
 			throws DWARFException, IOException, CancelledException {
 
 		long startOffset = debugInfoBR.getPointerIndex();
-		long length = debugInfoBR.readNextUnsignedInt();
-		int format;
+		LengthResult lengthInfo =
+			DWARFUtil.readLength(debugInfoBR, dwarfProgram.getGhidraProgram());
 
-		if (length == 0xffffffffL) {
-			// Length of 0xffffffff implies 64-bit DWARF format
-			// Mostly untested as there is no easy way to force the compiler
-			// to generate this
-			length = debugInfoBR.readNextLong();
-			format = DWARF_64;
-		}
-		else if (length >= 0xfffffff0L) {
-			// Length of 0xfffffff0 or greater is reserved for DWARF
-			throw new DWARFException("Reserved DWARF length value: " + Long.toHexString(length) +
-				". Unknown extension.");
-		}
-		else if (length == 0) {
-			throw new DWARFException("Invalid length 0 for DWARF Compilation Unit at 0x" +
-				Long.toHexString(startOffset));
-		}
-		else {
-			format = DWARF_32;
-		}
-
-		long endOffset = (debugInfoBR.getPointerIndex() + length);
+		long endOffset = debugInfoBR.getPointerIndex() + lengthInfo.length;
 		short version = debugInfoBR.readNextShort();
-		long abbreviationOffset = DWARFUtil.readOffsetByDWARFformat(debugInfoBR, format);
+		long abbreviationOffset = DWARFUtil.readOffsetByDWARFformat(debugInfoBR, lengthInfo.format);
 		byte pointerSize = debugInfoBR.readNextByte();
 		long firstDIEOffset = debugInfoBR.getPointerIndex();
 
@@ -172,16 +153,16 @@ public class DWARFCompilationUnit {
 		Map<Integer, DWARFAbbreviation> abbrMap =
 			DWARFAbbreviation.readAbbreviations(debugAbbrBR, dwarfProgram, monitor);
 
-		DWARFCompilationUnit cu =
-			new DWARFCompilationUnit(dwarfProgram, startOffset, endOffset, length, format, version,
-				abbreviationOffset, pointerSize, cuNumber, firstDIEOffset, abbrMap);
+		DWARFCompilationUnit cu = new DWARFCompilationUnit(dwarfProgram, startOffset, endOffset,
+			lengthInfo.length, lengthInfo.format, version, abbreviationOffset, pointerSize,
+			cuNumber, firstDIEOffset, abbrMap);
 
 		try {
 			DebugInfoEntry compileUnitDIE =
 				DebugInfoEntry.read(debugInfoBR, cu, dwarfProgram.getAttributeFactory());
 
-			DWARFCompileUnit compUnit = DWARFCompileUnit.read(
-				DIEAggregate.createSingle(compileUnitDIE), dwarfProgram.getDebugLine());
+			DWARFCompileUnit compUnit =
+				DWARFCompileUnit.read(DIEAggregate.createSingle(compileUnitDIE));
 			cu.setCompileUnit(compUnit);
 			return cu;
 		}

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DWARFCompileUnit.java

@@ -18,7 +18,6 @@ package ghidra.app.util.bin.format.dwarf4;
 import java.io.File;
 import java.io.IOException;
 
-import ghidra.app.util.bin.BinaryReader;
 import ghidra.app.util.bin.format.dwarf4.encoding.*;
 
 /**
@@ -32,13 +31,12 @@ public class DWARFCompileUnit {
 	private final Number high_pc;
 	private final Number low_pc;
 	private final Number language;
-	private final Number stmt_list;
 	private final DWARFIdentifierCase identifier_case;
 	private final boolean hasDWO;
 
 	private DWARFLine line = null;
 
-	public static DWARFCompileUnit read(DIEAggregate diea, BinaryReader lineReader)
+	public static DWARFCompileUnit read(DIEAggregate diea)
 			throws IOException, DWARFException {
 		if (diea.getTag() != DWARFTag.DW_TAG_compile_unit) {
 			throw new IOException("Expecting a DW_TAG_compile_unit DIE, found " + diea.getTag());
@@ -48,7 +46,7 @@ public class DWARFCompileUnit {
 		String producer = diea.getString(DWARFAttribute.DW_AT_producer, null);
 		String comp_dir = diea.getString(DWARFAttribute.DW_AT_comp_dir, null);
 
-		Number high_pc = null, low_pc = null, language = null, stmt_list = null;
+		Number high_pc = null, low_pc = null, language = null;
 
 		if (diea.hasAttribute(DWARFAttribute.DW_AT_low_pc)) {
 			low_pc = diea.getLowPC(0);
@@ -64,11 +62,6 @@ public class DWARFCompileUnit {
 			language = diea.getUnsignedLong(DWARFAttribute.DW_AT_language, -1);
 		}
 
-		// DW_AT_stmt_list can be const or ptr form types.
-		if (diea.hasAttribute(DWARFAttribute.DW_AT_stmt_list)) {
-			stmt_list = diea.getUnsignedLong(DWARFAttribute.DW_AT_stmt_list, -1);
-		}
-
 		DWARFIdentifierCase identifier_case = null;
 		if (diea.hasAttribute(DWARFAttribute.DW_AT_identifier_case)) {
 			identifier_case = DWARFIdentifierCase.find(
@@ -78,13 +71,9 @@ public class DWARFCompileUnit {
 		boolean hasDWO = diea.hasAttribute(DWARFAttribute.DW_AT_GNU_dwo_id) &&
 			diea.hasAttribute(DWARFAttribute.DW_AT_GNU_dwo_name);
 
-		DWARFLine line = null;
-		if (stmt_list != null && lineReader != null) {
-			lineReader.setPointerIndex(stmt_list.longValue());
-			line = new DWARFLine(lineReader);
-		}
+		DWARFLine line = DWARFLine.read(diea);
 
-		return new DWARFCompileUnit(name, producer, comp_dir, low_pc, high_pc, language, stmt_list,
+		return new DWARFCompileUnit(name, producer, comp_dir, low_pc, high_pc, language,
 			identifier_case, hasDWO, line);
 	}
 
@@ -92,15 +81,14 @@ public class DWARFCompileUnit {
 	 * Construct a DWARF compile unit with the given values.
 	 */
 	public DWARFCompileUnit(String name, String producer, String comp_dir, Number low_pc,
-			Number high_pc, Number language, Number stmt_list, DWARFIdentifierCase identifier_case,
-			boolean hasDWO, DWARFLine line) {
+			Number high_pc, Number language, DWARFIdentifierCase identifier_case, boolean hasDWO,
+			DWARFLine line) {
 		this.name = name;
 		this.producer = producer;
 		this.comp_dir = comp_dir;
 		this.low_pc = low_pc;
 		this.high_pc = high_pc;
 		this.language = language;
-		this.stmt_list = stmt_list;
 		this.identifier_case = identifier_case;
 		this.hasDWO = hasDWO;
 		this.line = line;
@@ -199,14 +187,6 @@ public class DWARFCompileUnit {
 		return this.language == null ? -1 : this.language.intValue();
 	}
 
-	/**
-	 * Get the statement list of the compile unit.
-	 * @return the statement list of the compile unit
-	 */
-	public int getStatementList() {
-		return this.stmt_list == null ? -1 : this.stmt_list.intValue();
-	}
-
 	/**
 	 * Get the identifier case of the compile unit
 	 * @return the identifier case of the compile unit
@@ -223,7 +203,6 @@ public class DWARFCompileUnit {
 	public String toString() {
 		return "DWARFCompileUnit [name=" + name + ", producer=" + producer + ", comp_dir=" +
 			comp_dir + ", high_pc=" + high_pc + ", low_pc=" + low_pc + ", language=" + language +
-			", stmt_list=" + stmt_list + ", identifier_case=" + identifier_case + ", hasDWO=" +
-			hasDWO + ", line=" + line + "]";
+			", identifier_case=" + identifier_case + ", hasDWO=" + hasDWO + ", line=" + line + "]";
 	}
 }

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DWARFLine.java

@@ -23,6 +23,9 @@ import java.util.List;
 import org.apache.commons.io.FilenameUtils;
 
 import ghidra.app.util.bin.BinaryReader;
+import ghidra.app.util.bin.format.dwarf4.DWARFUtil.LengthResult;
+import ghidra.app.util.bin.format.dwarf4.encoding.DWARFAttribute;
+import ghidra.app.util.bin.format.dwarf4.next.DWARFProgram;
 
 public class DWARFLine {
 	private long unit_length;
@@ -39,63 +42,80 @@ public class DWARFLine {
 	private List<String> include_directories;
 	private List<DWARFFile> file_names;
 
-	// TODO: convert this to a static factory method and a simple setter ctor
-	public DWARFLine(BinaryReader reader) throws IOException, DWARFException {
-		this.unit_length = reader.readNextUnsignedInt();
-		// Length of 0xffffffff implies 64-bit DWARF format
-		if (this.unit_length == 0xffffffffL) {
-			this.unit_length = reader.readNextLong();
-			this.format = DWARFCompilationUnit.DWARF_64;
-		}
-		// Length of 0xfffffff0 or greater is reserved for DWARF
-		else if (this.unit_length >= 0xfffffff0L) {
-			throw new DWARFException("Reserved DWARF length value: " +
-				Long.toHexString(this.unit_length) + ". Unknown extension.");
-		}
-		else {
-			this.format = DWARFCompilationUnit.DWARF_32;
+	/**
+	 * Read a DWARFLine from the compile unit's DW_AT_stmt_list location in the 
+	 * DebugLine stream (if present).
+	 * 
+	 * @param diea {@link DIEAggregate} compile unit DIE(a) 
+	 * @return a new DWARFLine instance if DW_AT_stmt_list & stream are present, otherwise null
+	 * @throws IOException if error reading data
+	 * @throws DWARFException if bad DWARF values
+	 */
+	public static DWARFLine read(DIEAggregate diea) throws IOException, DWARFException {
+		DWARFProgram dProg = diea.getProgram();
+		BinaryReader reader = dProg.getDebugLine();
+
+		// DW_AT_stmt_list can be const or ptr form types.
+		long stmtListOffset = diea.getUnsignedLong(DWARFAttribute.DW_AT_stmt_list, -1);
+
+		if (reader == null || stmtListOffset < 0) {
+			return null;
 		}
 
+		reader.setPointerIndex(stmtListOffset);
+
+		DWARFLine result = new DWARFLine();
+
+		LengthResult lengthInfo = DWARFUtil.readLength(reader, dProg.getGhidraProgram());
+		result.unit_length = lengthInfo.length;
+		result.format = lengthInfo.format;
+
 		// A version number for this line number information section
-		this.version = reader.readNextUnsignedShort();
+		result.version = reader.readNextUnsignedShort();
 
 		// Get the header length based on the current format
-		this.header_length = DWARFUtil.readOffsetByDWARFformat(reader, this.format);
+		result.header_length = DWARFUtil.readOffsetByDWARFformat(reader, result.format);
 
-		this.minimum_instruction_length = reader.readNextUnsignedByte();
+		result.minimum_instruction_length = reader.readNextUnsignedByte();
 
 		// Maximum operations per instruction only exists in DWARF version 4 or higher
-		if (this.version >= 4) {
-			this.maximum_operations_per_instruction = reader.readNextUnsignedByte();
+		if (result.version >= 4) {
+			result.maximum_operations_per_instruction = reader.readNextUnsignedByte();
 		}
 		else {
-			this.maximum_operations_per_instruction = 1;
+			result.maximum_operations_per_instruction = 1;
 		}
-		this.default_is_stmt = reader.readNextUnsignedByte();
-		this.line_base = reader.readNextByte();
-		this.line_range = reader.readNextUnsignedByte();
-		this.opcode_base = reader.readNextUnsignedByte();
-		this.standard_opcode_length = new int[this.opcode_base];
-		this.standard_opcode_length[0] = 1; /* Should never be used */
-		for (int i = 1; i < this.opcode_base; i++) {
-			this.standard_opcode_length[i] = reader.readNextUnsignedByte();
+		result.default_is_stmt = reader.readNextUnsignedByte();
+		result.line_base = reader.readNextByte();
+		result.line_range = reader.readNextUnsignedByte();
+		result.opcode_base = reader.readNextUnsignedByte();
+		result.standard_opcode_length = new int[result.opcode_base];
+		result.standard_opcode_length[0] = 1; /* Should never be used */
+		for (int i = 1; i < result.opcode_base; i++) {
+			result.standard_opcode_length[i] = reader.readNextUnsignedByte();
 		}
 
 		// Read all include directories
-		this.include_directories = new ArrayList<>();
+		result.include_directories = new ArrayList<>();
 		String include = reader.readNextAsciiString();
 		while (include.length() != 0) {
-			this.include_directories.add(include);
+			result.include_directories.add(include);
 			include = reader.readNextAsciiString();
 		}
 
 		// Read all files
-		this.file_names = new ArrayList<>();
+		result.file_names = new ArrayList<>();
 		DWARFFile file = new DWARFFile(reader);
 		while (file.getName().length() != 0) {
-			this.file_names.add(file);
+			result.file_names.add(file);
 			file = new DWARFFile(reader);
 		}
+
+		return result;
+	}
+
+	private DWARFLine() {
+		// empty, use #read()
 	}
 
 	/**

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DWARFUtil.java

@@ -30,6 +30,7 @@ import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag;
 import ghidra.app.util.bin.format.dwarf4.next.DWARFProgram;
 import ghidra.program.model.data.DataType;
 import ghidra.program.model.data.DataTypeComponent;
+import ghidra.program.model.listing.Program;
 import ghidra.program.model.symbol.SymbolType;
 import ghidra.util.Conv;
 
@@ -567,4 +568,64 @@ public class DWARFUtil {
 		return (referers.size() == 1) ? referers.get(0) : null;
 	}
 
+	public static class LengthResult {
+		public final long length;
+		public final int format;	// either DWARF_32 or DWARF_64
+
+		private LengthResult(long length, int format) {
+			this.length = length;
+			this.format = format;
+		}
+	}
+
+	/**
+	 * Read a variable-length length value from the stream.
+	 * <p>
+	 * 
+	 * @param reader {@link BinaryReader} stream to read from
+	 * @param program Ghidra {@link Program} 
+	 * @return new {@link LengthResult}, never null
+	 * @throws IOException if io error
+	 * @throws DWARFException if invalid values
+	 */
+	public static LengthResult readLength(BinaryReader reader, Program program)
+			throws IOException, DWARFException {
+		long startOffset = reader.getPointerIndex();
+		long length = reader.readNextUnsignedInt();
+		int format;
+
+		if (length == 0xffffffffL) {
+			// Length of 0xffffffff implies 64-bit DWARF format
+			// Mostly untested as there is no easy way to force the compiler
+			// to generate this
+			length = reader.readNextLong();
+			format = DWARFCompilationUnit.DWARF_64;
+		}
+		else if (length >= 0xfffffff0L) {
+			// Length of 0xfffffff0 or greater is reserved for DWARF
+			throw new DWARFException("Reserved DWARF length value: " + Long.toHexString(length) +
+				". Unknown extension.");
+		}
+		else if (length == 0) {
+			// Test for special case of weird BE MIPS 64bit length value.
+			// Instead of following DWARF std (a few lines above with length == MAX_INT),
+			// it writes a raw 64bit long (BE). The upper 32 bits (already read as length) will 
+			// always be 0 since super-large binaries from that system weren't really possible.
+			// The next 32 bits will be the remainder of the value.
+			if ( reader.isBigEndian() && program.getDefaultPointerSize() == 8) {
+				length = reader.readNextUnsignedInt();
+				format = DWARFCompilationUnit.DWARF_64;
+			}
+			else {
+				throw new DWARFException(
+					"Invalid DWARF length 0 at 0x" + Long.toHexString(startOffset));
+			}
+		}
+		else {
+			format = DWARFCompilationUnit.DWARF_32;
+		}
+
+		return new LengthResult(length, format);
+	}
+
 }

Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/format/dwarf4/MockDWARFCompilationUnit.java

@@ -35,7 +35,7 @@ public class MockDWARFCompilationUnit extends DWARFCompilationUnit {
 
 		setCompileUnit(
 			new DWARFCompileUnit("Mock Comp Unit", "Mock Comp Unit Producer", "Mock Comp Unit Dir",
-				0, 0, language, 0, DWARFIdentifierCase.DW_ID_case_insensitive, false, null));
+				0, 0, language, DWARFIdentifierCase.DW_ID_case_insensitive, false, null));
 		compUnitDIE = new DIECreator(DWARFTag.DW_TAG_compile_unit)
 				.addString(DWARFAttribute.DW_AT_name, "MockCompUnit" + compUnitNumber)
 				.create(this);