From d6352a32472e28eb008f3134daaf1f83ee120605 Mon Sep 17 00:00:00 2001 From: James <49045138+ghidracadabra@users.noreply.github.com> Date: Thu, 1 Dec 2022 18:02:28 +0000 Subject: [PATCH 1/3] GP-2906_James_exhaust_function_interiors --- .../RandomForestTrainingTask.java | 9 ++++-- .../RandomForestTrainingTaskTest.java | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTask.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTask.java index a6c539b82f..7313c57f88 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTask.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTask.java @@ -277,15 +277,18 @@ public class RandomForestTrainingTask extends Task { monitor.setMessage( "Selecting " + numEntries * factor + " random addresses within function interiors"); start = System.nanoTime(); - AddressSetView randomFuncInteriors = - RandomSubsetUtils.randomSubset(selectableInteriors, numEntries * factor, monitor); + long numInteriors = numEntries * factor; + + AddressSetView randomFuncInteriors = numInteriors < selectableInteriors.getNumAddresses() + ? RandomSubsetUtils.randomSubset(selectableInteriors, numInteriors, monitor) + : selectableInteriors; end = System.nanoTime(); Msg.info(this, String.format("factor: %d elapsed selecting random interiors: %g seconds", factor, (end - start) / NANOSECONDS_PER_SECOND)); trainingNegative = trainingNegative.union(randomFuncInteriors); if (trainingNegative.isEmpty()) { Msg.showError(this, null, "Data Gathering Error", - "No function interiors in training set"); + "No non-starts in training set for sampling factor " + factor); return null; } if (trainingPositive.intersects(trainingNegative)) { diff --git a/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java b/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java index 97d0b94a10..277faf5298 100644 --- a/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java +++ b/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java @@ -358,4 +358,33 @@ public class RandomForestTrainingTaskTest extends AbstractProgramBasedTest { assertTrue(data.getTestNegative().contains(definedData)); } + @Test + public void testExhaustingFunctionInteriors() throws CancelledException { + params = new FunctionStartRFParams(program); + params.setMaxStarts(5); + int tooBig = 10; + Address begin = program.getSymbolTable().getSymbols("entry").next().getAddress(); + AddressSet entries = new AddressSet(); + for (int i = 0; i < 10; ++i) { + entries.add(begin.add(i)); + } + AddressSet interiors = new AddressSet(); + for (int i = 10; i < 25; ++i) { + interiors.add(begin.add(i)); + } + AddressSet definedData = new AddressSet(); + for (int i = 25; i < 30; ++i) { + definedData.add(begin.add(i)); + } + RandomForestTrainingTask task = new RandomForestTrainingTask(program, params, null, + RandomForestFunctionFinderPlugin.TEST_SET_MAX_SIZE_DEFAULT); + TrainingAndTestData data = + task.getTrainingAndTestData(entries, interiors, definedData, tooBig, TaskMonitor.DUMMY); + assertTrue(data.getTrainingPositive().getNumAddresses() == 5); + assertTrue(data.getTestPositive().getNumAddresses() == 5); + assertTrue(data.getTestPositive().union(data.getTrainingPositive()).equals(entries)); + assertTrue(data.getTrainingNegative().equals(interiors)); + assertTrue(data.getTestNegative().equals(definedData)); + } + } From 321c3455a342daee4d435b06eb7f3d32cbaea541 Mon Sep 17 00:00:00 2001 From: emteere <47253321+emteere@users.noreply.github.com> Date: Thu, 1 Dec 2022 19:33:53 -0500 Subject: [PATCH 2/3] GP-2878 Adjusted RISCV function start pattern totalbits and postbits and instruction flow / decode changes, don't decode 0x0000 as unimpl --- .../RISCV/data/languages/riscv.reg.sinc | 6 ++ .../RISCV/data/languages/riscv.rv32i.sinc | 24 +++++-- .../RISCV/data/languages/riscv.rvc.sinc | 12 ++-- .../RISCV/data/languages/riscv.table.sinc | 18 +++-- .../RISCV/data/languages/riscv32.cspec | 26 +++---- .../RISCV/data/languages/riscv64.cspec | 26 +++---- .../RISCV/data/patterns/riscv_gc_patterns.xml | 69 +++++++++---------- 7 files changed, 100 insertions(+), 81 deletions(-) diff --git a/Ghidra/Processors/RISCV/data/languages/riscv.reg.sinc b/Ghidra/Processors/RISCV/data/languages/riscv.reg.sinc index caefb8135c..5b09d07e43 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv.reg.sinc +++ b/Ghidra/Processors/RISCV/data/languages/riscv.reg.sinc @@ -709,7 +709,10 @@ define token instr (32) op2030=(20,30) op2031=(20,31) sop2031=(20,31) signed + op2121=(21,21) op2130=(21,30) + op2222=(22,22) + op2323=(23,23) op2324=(23,24) op2424=(24,24) op2427=(24,27) @@ -753,8 +756,10 @@ define token cinstr (16) cr0206=(2,6) cfr0206=(2,6) cop0212=(2,12) + cop0303=(3,3) cop0304=(3,4) cop0305=(3,5) + cop0404=(4,4) cop0406=(4,6) cop0505=(5,5) cop0506=(5,6) @@ -773,6 +778,7 @@ define token cinstr (16) cfr0711=(7,11) cop0712=(7,12) cop0808=(8,8) + cop0909=(9,9) cop0910=(9,10) cop0912=(9,12) cop1010=(10,10) diff --git a/Ghidra/Processors/RISCV/data/languages/riscv.rv32i.sinc b/Ghidra/Processors/RISCV/data/languages/riscv.rv32i.sinc index baaf5142fd..c93efef488 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv.rv32i.sinc +++ b/Ghidra/Processors/RISCV/data/languages/riscv.rv32i.sinc @@ -123,11 +123,18 @@ } # jal d,a 0000006f 0000007f JSR (0, 0) +# call if RA set, destreg == RA +:jal ra,immUJ is immUJ & ra & r0711=1 & op0001=0x3 & op0204=0x3 & op0506=0x3 +{ + ra = inst_next; + call immUJ; +} + +# goto if RA not set, destreg != RA :jal rd,immUJ is immUJ & rd & op0001=0x3 & op0204=0x3 & op0506=0x3 { - #TODO RAS: push inst_next if rd=x1/x5 rd = inst_next; - call immUJ; + goto immUJ; } # j a 0000006f 00000fff BRANCH|ALIAS (0, 0) @@ -136,14 +143,21 @@ goto immUJ; } - # jalr d,s,j 00000067 0000707f JSR (0, 0) +# call if RA set, destreg == RA +:jalr ra,rs1,immI is rs1 & immI & ra & r0711=1 & op0001=0x3 & op0204=0x1 & op0506=0x3 & funct3=0x0 +{ + local ea:$(XLEN) = (rs1 + immI) & ~1; + ra = inst_next; + call [ea]; +} + +# goto if RA not set, destreg != RA :jalr rd,rs1,immI is rs1 & immI & rd & op0001=0x3 & op0204=0x1 & op0506=0x3 & funct3=0x0 { - #TODO RAS: push inst_next/pop based on rd=x1/x5 or rs1=x1/x5 local ea:$(XLEN) = (rs1 + immI) & ~1; rd = inst_next; - call [ea]; + goto [ea]; } # jr o(s) 00000067 00007fff BRANCH|ALIAS (0, 0) diff --git a/Ghidra/Processors/RISCV/data/languages/riscv.rvc.sinc b/Ghidra/Processors/RISCV/data/languages/riscv.rvc.sinc index d82d4c44b6..480f1b6041 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv.rvc.sinc +++ b/Ghidra/Processors/RISCV/data/languages/riscv.rvc.sinc @@ -355,11 +355,13 @@ *[ram]:4 ea = crs2:4; } -# c.unimp 00000000 0000ffff SIMPLE (0, 0) -:c.unimp is cop0001=0x0 & cop1315=0x0 & cop0212=0x0 -{ - trap(); -} +# c.unimp 00000000 0000ffff SIMPLE (0, 0) +# would be better not to decode as it is used as padding +# +# :c.unimp is cop0001=0x0 & cop1315=0x0 & cop0212=0x0 +#{ +# trap(); +#} # c.xor Cs,Ct 00008c21 0000fc63 SIMPLE (0, 0) :c.xor cr0709s,cr0204s is cr0204s & cr0709s & cop0001=0x1 & cop1315=0x4 & cop0506=0x1 & cop1012=0x3 diff --git a/Ghidra/Processors/RISCV/data/languages/riscv.table.sinc b/Ghidra/Processors/RISCV/data/languages/riscv.table.sinc index 92a0c90fae..749ac3e8e4 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv.table.sinc +++ b/Ghidra/Processors/RISCV/data/languages/riscv.table.sinc @@ -203,21 +203,25 @@ cbimm: reloc is scop1212 & cop1011 & cop0506 & cop0304 & cop0202 [ reloc = inst_ # used for goto cjimm: reloc is scop1212 & cop1111 & cop0910 & cop0808 & cop0707 & cop0606 & cop0305 & cop0202 [ reloc = inst_start + ((scop1212 << 11) | (cop1111 << 4) | (cop0910 << 8) | (cop0808 << 10) | (cop0707 << 6) | (cop0606 << 7) | (cop0305 << 1) | (cop0202 << 5)); ] { export *[ram]:$(XLEN) reloc; } + +nzuimm5: is cop0606=1 | cop0505=1 | cop0404=1 | cop0303 = 1 | cop0202=1 {} +nzuimm6: is cop1212=1 | cop0606=1 | cop0505=1 | cop0404=1 | cop0303 = 1 | cop0202=1 {} + @if ADDRSIZE == "32" -#TODO nonzero, would like to have cop0205>0 -c6imm: uimm is cop1212=0 & cop0206 [ uimm = (cop0206 + 0); ] { local tmp:$(XLEN) = uimm; export tmp; } +c6imm: uimm is cop1212=0 & cop0206 & nzuimm5 [ uimm = (cop0206 + 0); ] { local tmp:$(XLEN) = uimm; export tmp; } @elif ADDRSIZE == "64" -#TODO nonzero, would like to have cop0205>0 | cop0206>0 -c6imm: uimm is cop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm; export tmp; } +c6imm: uimm is cop1212 & cop0206 & nzuimm6 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm; export tmp; } @elif ADDRSIZE == "128" c6imm: uimm is cop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm + (64 * (uimm == 0)); export tmp; } @endif -cbigimm: uimm is cop1212 & scop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = (scop1212 << 17) | (cop0206 << 12); export tmp; } +cbigimm: uimm is cop1212 & scop1212 & cop0206 & nzuimm6 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = (scop1212 << 17) | (cop0206 << 12); export tmp; } -caddi4spnimm: uimm is cop1112 & cop0710 & cop0606 & cop0505 [ uimm = (cop0710 << 6) | (cop1112 << 4) | (cop0505 << 3) | (cop0606 << 2); ] { local tmp:$(XLEN) = uimm; export tmp; } +nzcaddi4: is cop1212=1 | cop1111=1 | cop1010=1 | cop0909=1 | cop0808=1 | cop0707=1 | cop0606=1 | cop0505=1 {} -caddi16spimm: imm is scop1212 & cop0606 & cop0505 & cop0304 & cop0202 [ imm = (scop1212 << 9) | (cop0304 << 7) | (cop0505 << 6) | (cop0202 << 5) | (cop0606 << 4); ] { local tmp:$(XLEN) = imm; export tmp; } +caddi4spnimm: uimm is nzcaddi4 & cop1112 & cop0710 & cop0606 & cop0505 [ uimm = (cop0710 << 6) | (cop1112 << 4) | (cop0505 << 3) | (cop0606 << 2); ] { local tmp:$(XLEN) = uimm; export tmp; } + +caddi16spimm: imm is scop1212 & cop0606 & cop0505 & cop0304 & cop0202 & nzuimm6 [ imm = (scop1212 << 9) | (cop0304 << 7) | (cop0505 << 6) | (cop0202 << 5) | (cop0606 << 4); ] { local tmp:$(XLEN) = imm; export tmp; } clwimm: uimm is cop1012 & cop0606 & cop0505 [ uimm = (cop1012 << 3) | (cop0606 << 2) | (cop0505 << 6); ] { local tmp:$(XLEN) = uimm; export tmp; } diff --git a/Ghidra/Processors/RISCV/data/languages/riscv32.cspec b/Ghidra/Processors/RISCV/data/languages/riscv32.cspec index 86b26740cf..fa59edcfee 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv32.cspec +++ b/Ghidra/Processors/RISCV/data/languages/riscv32.cspec @@ -69,20 +69,20 @@ - - + + - - - - - - - - - - - + + + + + + + + + + + diff --git a/Ghidra/Processors/RISCV/data/languages/riscv64.cspec b/Ghidra/Processors/RISCV/data/languages/riscv64.cspec index e99c201d56..20eab6a2c3 100644 --- a/Ghidra/Processors/RISCV/data/languages/riscv64.cspec +++ b/Ghidra/Processors/RISCV/data/languages/riscv64.cspec @@ -69,20 +69,20 @@ - - + + - - - - - - - - - - - + + + + + + + + + + + diff --git a/Ghidra/Processors/RISCV/data/patterns/riscv_gc_patterns.xml b/Ghidra/Processors/RISCV/data/patterns/riscv_gc_patterns.xml index 14ecd69948..d189d6d86f 100644 --- a/Ghidra/Processors/RISCV/data/patterns/riscv_gc_patterns.xml +++ b/Ghidra/Processors/RISCV/data/patterns/riscv_gc_patterns.xml @@ -1,52 +1,45 @@ - + + + - - 10000010 10000000 - ......01 101..... + 10000010 10000000 + 10000010 10000000 00000000 00000000 - - 0.....01 01110001 - 0.....01 00010001 - - - - - - - 10000010 10000000 - ......01 101..... - - - + 0.....01 01110001 + 0.....01 00010001 00010011 00000001 ....0001 1....... - .0010111 ........ ........ ........ - + + + - - 01101111 ....0000 ........ ........ + 01101111 ....0000 ........ ........ + ......01 101..... + ......01 101..... 00000000 00000000 - - 0.....01 01110001 - 0.....01 00010001 - - - - - - - 01101111 ....0000 ........ ........ - - - + 0.....01 01110001 + 0.....01 00010001 00010011 00000001 ....0001 1....... - .0010111 ........ ........ ........ - + - + + + + + 10000010 10000000 + 10000010 10000000 00000000 00000000 + ......01 101..... 00000000 00000000 + 01101111 ....0000 ........ ........ + + + .0010111 ........ ........ ........ + + + + \ No newline at end of file From 9aa6b6aa6be2345e3d705f96ac38887924fb0c79 Mon Sep 17 00:00:00 2001 From: emteere <47253321+emteere@users.noreply.github.com> Date: Fri, 2 Dec 2022 00:41:23 +0000 Subject: [PATCH 3/3] GP-2910 Add volatile range tag to 8051 bit mapped registers --- Ghidra/Processors/8051/certification.manifest | 6 +++--- .../Processors/8051/data/languages/80251.pspec | 1 + .../Processors/8051/data/languages/8051.pspec | 1 + .../Processors/8051/data/languages/mx51.pspec | 17 +++++++++-------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Ghidra/Processors/8051/certification.manifest b/Ghidra/Processors/8051/certification.manifest index fc6cfe4588..7e2d6acc9e 100644 --- a/Ghidra/Processors/8051/certification.manifest +++ b/Ghidra/Processors/8051/certification.manifest @@ -1,19 +1,19 @@ ##VERSION: 2.0 Module.manifest||GHIDRA||||END| data/languages/80251.cspec||GHIDRA||||END| -data/languages/80251.pspec||GHIDRA||reviewed||END| +data/languages/80251.pspec||GHIDRA||||END| data/languages/80251.sinc||GHIDRA||||END| data/languages/80251.slaspec||GHIDRA||||END| data/languages/80390.cspec||GHIDRA||||END| data/languages/80390.slaspec||GHIDRA||||END| data/languages/8051.cspec||GHIDRA||||END| data/languages/8051.ldefs||GHIDRA||||END| -data/languages/8051.pspec||GHIDRA||reviewed||END| +data/languages/8051.pspec||GHIDRA||||END| data/languages/8051.slaspec||GHIDRA||||END| data/languages/8051_archimedes.cspec||GHIDRA||||END| data/languages/8051_main.sinc||GHIDRA||||END| data/languages/mx51.cspec||GHIDRA||||END| -data/languages/mx51.pspec||GHIDRA||reviewed||END| +data/languages/mx51.pspec||GHIDRA||||END| data/languages/mx51.sinc||GHIDRA||||END| data/languages/mx51.slaspec||GHIDRA||||END| data/languages/old/8051v1.lang||GHIDRA||reviewed||END| diff --git a/Ghidra/Processors/8051/data/languages/80251.pspec b/Ghidra/Processors/8051/data/languages/80251.pspec index 3d950cc6ad..a6b68b5160 100644 --- a/Ghidra/Processors/8051/data/languages/80251.pspec +++ b/Ghidra/Processors/8051/data/languages/80251.pspec @@ -6,6 +6,7 @@ + diff --git a/Ghidra/Processors/8051/data/languages/8051.pspec b/Ghidra/Processors/8051/data/languages/8051.pspec index a80d562650..8f2fbadd27 100644 --- a/Ghidra/Processors/8051/data/languages/8051.pspec +++ b/Ghidra/Processors/8051/data/languages/8051.pspec @@ -6,6 +6,7 @@ + diff --git a/Ghidra/Processors/8051/data/languages/mx51.pspec b/Ghidra/Processors/8051/data/languages/mx51.pspec index 9e3dfd4f43..e5f4f3d62f 100644 --- a/Ghidra/Processors/8051/data/languages/mx51.pspec +++ b/Ghidra/Processors/8051/data/languages/mx51.pspec @@ -6,6 +6,7 @@ + @@ -47,10 +48,10 @@ - - - - + + + + @@ -274,10 +275,10 @@ - - - - +