From def5113ba66b9861cff870772cf11d78a456ad35 Mon Sep 17 00:00:00 2001 From: adamopolous Date: Tue, 4 Jun 2019 15:21:06 -0400 Subject: [PATCH 1/9] GT-2897: Gradle installation script for external dependencies --- DevGuide.md | 26 +++ gradle/certification.manifest | 1 + gradle/init.gradle | 357 ++++++++++++++++++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 gradle/init.gradle diff --git a/DevGuide.md b/DevGuide.md index 142630a455..62cc99ac3f 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -71,6 +71,30 @@ Ghidra's build uses artifacts named as available in Maven Central and Bintray JC Unfortunately, in some cases, the artifact or the particular version we desire is not available. So, in addition to mavenCentral and jcenter, you must configure a flatDir-style repository for manually-downloaded dependencies. +#### Dependency Setup Script +The flat repository mentioned above can be done automatically by running a simple Gradle script. +Navigate to the Ghidra clone you just created and from the top-level ghidra folder, run the following: +``` +gradle --init-script gradle/init.gradle tasks +``` +The Gradle task to be executed, in this case _tasks_, is unimportant. The point is to have Gradle execute +the __init.gradle__ script. If it ran correctly you will have a new folder, __flatRepo/__, in your home directory populated with the following jar files: + * AXMLPrinter2 + * csframework + * dex-ir-2.0 + * dex-reader-2.0 + * dex-reader-api-2.0 + * dex-tools-2.0 + * dex-translator-2.0 + * dex-writer-2.0 + * hfsx + * hfsx_dmglib + * iharder-base64 + +There will also be a new archive, yajsw-stable-12.12.zip, placed in __ghidra.bin/Ghidra/Features/GhidraServer/__. + +If you see these, congrats! Skip to [importing the Gradle project](#import-gradle-project). If not, continue with manual configuration below... + Create `~/.gradle/init.d/repos.gradle` with the following contents: ```groovy @@ -216,6 +240,8 @@ To build the full Ghidra distribution, you must also build the GhidraServer. ## Get Dependencies for GhidraServer +_Note_: If you already ran the [dependency setup script](#dependency-setup-script), you can skip this section and continue on to [building the package](#building-the-package). + Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in a directory named: `ghidra.bin/Ghidra/Features/GhidraServer/`. Note that `ghidra.bin` must be a sibling of `ghidra`: diff --git a/gradle/certification.manifest b/gradle/certification.manifest index bf7e957f08..fe96782ed1 100644 --- a/gradle/certification.manifest +++ b/gradle/certification.manifest @@ -5,6 +5,7 @@ distributableGhidraExtension.gradle||GHIDRA||||END| distributableGhidraModule.gradle||GHIDRA||||END| externalGhidraExtension.gradle||GHIDRA||||END| helpProject.gradle||GHIDRA||||END| +init.gradle||GHIDRA||||END| jacocoProject.gradle||GHIDRA||||END| javaProject.gradle||GHIDRA||||END| javaTestProject.gradle||GHIDRA||||END| diff --git a/gradle/init.gradle b/gradle/init.gradle new file mode 100644 index 0000000000..adc52ecd78 --- /dev/null +++ b/gradle/init.gradle @@ -0,0 +1,357 @@ +/******************************************************************************* + * init.gradle * + * * + * Sets up the gradle configuration for external users and downloads * + * any required dependencies that aren't available in the * + * other online repositories (eg: maven). This should be run * + * immediately after cloning the Ghidra repository before any other gradle * + * tasks are run. * + * * + * Specifically, this task: * + * * + * 1. Downloads various jars required by the ghidra build and * + * puts them in /flatRepo. The jars to be * + * downloaded: * + * - dex-tools-2.0.zip * + * - AXMLPrinter2.jar * + * - hfsexplorer-0_21-bin.zip * + * - yajsw-stable-12.12.zip (placed in GhidraServer location) * + * * + * 2. Creates a gradle configuration file (repos.config) in * + * /.gradle/init.d/. This contains repository * + * information used by gradle to find dependencies (it points * + * gradle to the flatRepo location created above). * + * * + * usage: from the command line in the main ghidra repository * + * directory, run the following: * + * * + * gradle --init-script gradle/init.gradle * + * * + * Note: Running this script multiple times will cause the config * + * file to be recreated and all dependencies re-downloaded. * + * * + * Note: All files are downloaded to a the standard java temporary folder * + * location, in a sub-folder called 'ghidra. This is cleaned up and * + * removed when the script completes. * + * TODO: make sure this folder is cleaned up in EVERY case, especially * + * if the script fails at some point * + * * + *******************************************************************************/ + +import java.util.zip.*; +import java.nio.file.*; +import java.security.MessageDigest; +import org.apache.commons.io.*; +import org.apache.commons.io.filefilter.*; + +ext.HOME_DIR = System.getProperty('user.home') +ext.FLAT_REPO_DIR = new File(HOME_DIR + "/flatRepo") +ext.TMP_FILE = File.createTempFile("downloads", ".tmp", null) +File TMP_DIR = TMP_FILE.getParentFile() +ext.DOWNLOADS_DIR = new File(TMP_DIR, "ghidra") + +// The URLs for each of the archives to be downloaded +ext.DEX_ZIP = 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip' +ext.AXML_ZIP = 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar' +ext.HFS_ZIP = 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip' +ext.YAJSW_ZIP = 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip' + +// Store the MD5s for each of the downloads so we can verify that we retrieved them +// all successfully +ext.DEX_MD5 = '032456b9db9e6059376611553aecf31f' +ext.AXML_MD5 = '55d70be9862c2b456cc91a933c197934' +ext.HFS_MD5 = 'cc1713d634d2cd1fd7f21e18ae4d5d5c' +ext.YAJSW_MD5 = 'e490ea92554f0238d74d4ef6161cb2c7' + +// Number of times to try and establish a connection when downloading files before +// failing +ext.NUM_RETRIES = 1 + +initscript { + repositories { + mavenCentral() + } + dependencies { + classpath 'commons-io:commons-io:2.5' + } +} + +try { + createDirs() + createConfigFile() + populateFlatRepo() +} +finally { + cleanup() +} + +/** + * Creates the directories where the dependencies will be downloaded and stored + */ +def createDirs() { + if (!DOWNLOADS_DIR.exists()) { + DOWNLOADS_DIR.mkdirs() + } + if (!FLAT_REPO_DIR.exists()) { + FLAT_REPO_DIR.mkdirs() + } +} + +/** + * Creates the repos.gradle configuration file that tells Gradle + * where to look for dependencies. This ensures that Gradle will + * find the jars we store in the local flat repo. + */ +def createConfigFile() { + + ext.repoConfigDir = new File(HOME_DIR + "/.gradle/init.d") + ext.repoConfigFile = new File(repoConfigDir, "repos.gradle") + + if (!repoConfigDir.exists()) { + repoConfigDir.mkdirs() + } + + repoConfigFile.write("ext.HOME = System.getProperty('user.home')") + repoConfigFile.append("\nallprojects {") + repoConfigFile.append("\nrepositories {") + repoConfigFile.append("\nmavenCentral()") + repoConfigFile.append("\njcenter()") + repoConfigFile.append('\nflatDir name: "flat", dirs:["$HOME/flatRepo"]') + repoConfigFile.append("\n}") + repoConfigFile.append("\n}") +} + +/** + * Downloads a file from a URL. If there is a problem connecting to the given + * URL the attempt will be retried NUM_RETRIES times before failing. + * + * Progress is shown on the command line in the form of the number of bytes + * downloaded; the total size of the file to download is not known during the + * download so no percentage of the total is given. + * + * @param url the file to download + * @param filename the local file to create for the download + */ +def download(url, filename) { + + BufferedInputStream istream = establishConnection(url, NUM_RETRIES); + assert istream != null : "***CONNECTION FAILURE***\nmax attempts exceeded; exiting\n" + + FileOutputStream ostream = new FileOutputStream(filename); + def dataBuffer = new byte[1024]; + int bytesRead; + int totalRead; + while ((bytesRead = istream.read(dataBuffer, 0, 1024)) != -1) { + + ostream.write(dataBuffer, 0, bytesRead); + totalRead += bytesRead + + // print progress on the same line in the console... + print("\r") + print("Downloading: " + filename + " " + totalRead) + System.out.flush() + } + println("") + + istream.close(); + ostream.close(); +} + +/** + * Attemps to establish a connection to the given URL. This will attempt to retry the + * connection in the event of a failure. + * + * @param url the site to connect to + * @param retries the number of times to attempt to reconnect if there is a failure + */ +def establishConnection(url, retries) { + println("Download file: " + url) + for (int i=0; i + (e.name as File).with { f -> + if (f.parentFile != null) { + File destPath = new File(targetDir.path, f.parentFile.path) + destPath.mkdirs() + File targetFile = new File(destPath.path, f.name) + targetFile.withOutputStream { w -> + w << zip.getInputStream(e) + } + } + } + } +} + +/** + * Downloads and stores the necessary dependencies in the local + * flat repository. + */ +def populateFlatRepo() { + + // 1. Download all the dependencies. + download (DEX_ZIP, DOWNLOADS_DIR.path + '/dex-tools-2.0.zip') + download (AXML_ZIP, FLAT_REPO_DIR.path + '/AXMLPrinter2.jar') + download (HFS_ZIP, DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip') + download (YAJSW_ZIP, DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip') + + validateChecksum(DOWNLOADS_DIR.path + '/dex-tools-2.0.zip', DEX_MD5); + validateChecksum(FLAT_REPO_DIR.path + '/AXMLPrinter2.jar', AXML_MD5); + validateChecksum(DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip', HFS_MD5); + validateChecksum(DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip', YAJSW_MD5); + + // 2. Unzip the dependencies; for some of these we don't need everything in + // the download so unzip them and only copy what we need. + // + unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex-tools-2.0.zip") + unzipHfsx() + unzipYajsw() + + // 3. Copy the necessary jars to the flatRepo directory. Yajsw is the + // exception; it needs to go in a GhidraServer folder. + copyDexTools() + copyHfsx() + copyYajsw() +} + +/** + * Generates the md5 for the given file and compares it against the + * expected result. If there is no match an assert exception will be + * generated. + * + * @param filename the fully-qualified file path+name + * @param expectedMd5 the expected md5 for the file + */ +def validateChecksum(filename, expectedMd5) { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(Paths.get(filename))); + byte[] digest = md.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + assert(sb.toString().equals(expectedMd5)); +} + +/** + * Unzips the hfsx zip file + */ +def unzipHfsx() { + def hfsxdir = getOrCreateTempHfsxDir() + unzip (DOWNLOADS_DIR, hfsxdir, "hfsexplorer-0_21-bin.zip") +} + +/** + * Unzips the yajsw zip file + */ +def unzipYajsw() { + def yajswdir = getOrCreateTempYajswDir() + unzip (DOWNLOADS_DIR, yajswdir, "yajsw-stable-12.12.zip") +} + +/** + * Copies the dex-tools jars to the flat repository + * + * Note: This will only copy files beginning with "dex-" + */ +def copyDexTools() { + FileUtils.copyDirectory(new File(DOWNLOADS_DIR, 'dex2jar-2.0/lib/'), FLAT_REPO_DIR, new WildcardFileFilter("dex-*")); +} + +/** + * Copies the hfsx jars to the flat repository + */ +def copyHfsx() { + FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/csframework.jar"), new File(FLAT_REPO_DIR, "csframework.jar")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx_dmglib.jar"), new File(FLAT_REPO_DIR, "hfsx_dmglib.jar")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx.jar"), new File(FLAT_REPO_DIR, "hfsx.jar")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/iharder-base64.jar"), new File(FLAT_REPO_DIR, "iharder-base64.jar")); +} + +/** + * Copies the yajswdir zip to its location in the GhidraServer project. + * + * Note: There is a Gradle task to do all of this (yajswDevUnpack) but we cannot execute + * that from here so just copy everything manually + */ +def copyYajsw() { + def userdir = System.getProperty("user.dir") + def yajswdir = new File(getOrCreateTempYajswDir(),"yajsw-stable-12.12") + def yajswdirTarget = new File(userdir, "/../ghidra/Ghidra/Features/GhidraServer/build/data/yajsw-stable-12.12") + + FileUtils.copyFile(new File(yajswdir, "yajsw.policy.txt"), new File(yajswdirTarget, "yajsw.policy.txt")); + FileUtils.copyFile(new File(yajswdir, "LICENSE.txt"), new File(yajswdirTarget, "LICENSE.txt")); + FileUtils.copyDirectory(yajswdir, yajswdirTarget, new WildcardFileFilter("*.jar")); + FileUtils.copyDirectory(new File(yajswdir, "templates"), new File(yajswdirTarget, "templates")); + FileUtils.copyDirectory(new File(yajswdir, "lib/extended"), new File(yajswdirTarget, "lib/extended")); + FileUtils.copyDirectory(new File(yajswdir, "doc"), new File(yajswdirTarget, "doc")); + FileUtils.copyDirectory(new File(yajswdir, "lib/core"), new File(yajswdirTarget, "lib/core")); +} + +/** + * Creates a temporary folder to house the hfsx zip contents + * + * @return the newly-created hfsx directory object + */ +def getOrCreateTempHfsxDir() { + def hfsxdir = new File (DOWNLOADS_DIR, "hfsx") + if (!hfsxdir.exists()) { + hfsxdir.mkdir() + } + + return hfsxdir; +} + +/** + * Creates a temporary folder to house the yajsw zip contents + */ +def getOrCreateTempYajswDir() { + def yajswdir = new File (DOWNLOADS_DIR, "yajsw") + if (!yajswdir.exists()) { + yajswdir.mkdir() + } + + return yajswdir; +} + +/** + * Performs any cleanup operations that need to be performed after the flat repo has + * been populated. + */ +def cleanup() { + remove(DOWNLOADS_DIR) + remove(TMP_FILE) +} + +/** + * Deletes the given file (or directory) + * + * @param fileOrDirectory the item (either a file or directory) to be deleted + */ +def remove(fileOrDirectory) { + if (fileOrDirectory.isDirectory()) { + for (File child : fileOrDirectory.listFiles()) { + remove(child); + } + } + + fileOrDirectory.delete(); +} \ No newline at end of file From e6e1302854e5ad8f5c18b01948259679e87a2e9b Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Mon, 24 Jun 2019 14:12:31 -0400 Subject: [PATCH 2/9] GT-2897: DevGuide reorg. --- DevGuide.md | 268 +++++++++++++++++++++++++--------------------------- 1 file changed, 127 insertions(+), 141 deletions(-) diff --git a/DevGuide.md b/DevGuide.md index 62cc99ac3f..f05b24525c 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -6,13 +6,16 @@ The following is a list of dependencies, in no particular order. This guide includes instructions for obtaining many of these at the relevant step(s). You may not need all of these, depending on which portions you are building or developing. -* JDK 11 - We test and build using OpenJDK 11.0.2. - - https://jdk.java.net/11/ +* Java JDK 11 - Free long term support (LTS) versions of JDK 11 are provided by: + - AdoptOpenJDK + - https://adoptopenjdk.net/releases.html?variant=openjdk11&jvmVariant=hotspot + - Amazon Corretto + - https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html * Eclipse - It must support JDK 11. Eclipse 2018-12 or later should work. Other IDEs may work, but we have not tested them. - https://www.eclipse.org/downloads/ * Gradle 5.0 - Later versions may work, but you'll need to modify our version check. - https://gradle.org/next-steps/?version=5.0&format=bin -* A C/C++ compiler - We use GCC on Linux, Xcode (Clang) on macOS, and Visual Studio 2017 on Windows, . +* A C/C++ compiler - We use GCC on Linux, Xcode (Clang) on macOS, and Visual Studio 2017 on Windows. - https://gcc.gnu.org/ - https://developer.apple.com/xcode/ - https://visualstudio.microsoft.com/downloads/ @@ -56,7 +59,7 @@ In Eclipse, select Window -> Prefereces (Eclipse -> Preferences on macOS), then Install Gradle, add it to your `PATH`, and ensure it is launched using JDK 11. -## Setup Source and Dependency Repositories +## Setup Source Repository You may choose any directory for your working copy, but these instructions will assume you have cloned the source to `~/git/ghidra`. Be sure to adjust the commands to match your chosen working directory if different than suggested: @@ -67,18 +70,26 @@ cd ~/git git clone git@github.com:NationalSecurityAgency/ghidra.git ``` +## Setup Build Dependency Repository + Ghidra's build uses artifacts named as available in Maven Central and Bintray JCenter, when possible. Unfortunately, in some cases, the artifact or the particular version we desire is not available. -So, in addition to mavenCentral and jcenter, you must configure a flatDir-style repository for manually-downloaded dependencies. +So, in addition to mavenCentral and jcenter, you must configure a flat directory-style repository for +manually-downloaded dependencies. -#### Dependency Setup Script -The flat repository mentioned above can be done automatically by running a simple Gradle script. -Navigate to the Ghidra clone you just created and from the top-level ghidra folder, run the following: +The flat directory-style repository can be created and populated automatically by a provided script, +or manually by downloading the required dependencies. Choose one of the two following methods: + * [Automatic script instructions](#automatic-script-instructions) + * [Manual download instructions](#manual-download_instructions) + +### Automatic Script Instructions +The flat directory-style repository can be setup automatically by running a simple Gradle script. +Navigate to the Ghidra clone directory you just created, and run the following: ``` gradle --init-script gradle/init.gradle tasks ``` The Gradle task to be executed, in this case _tasks_, is unimportant. The point is to have Gradle execute -the __init.gradle__ script. If it ran correctly you will have a new folder, __flatRepo/__, in your home directory populated with the following jar files: +the `init.gradle` script. If it ran correctly you will have a new folder, `flatRepo/`, in your home directory populated with the following jar files: * AXMLPrinter2 * csframework * dex-ir-2.0 @@ -93,7 +104,10 @@ the __init.gradle__ script. If it ran correctly you will have a new folder, __fl There will also be a new archive, yajsw-stable-12.12.zip, placed in __ghidra.bin/Ghidra/Features/GhidraServer/__. -If you see these, congrats! Skip to [importing the Gradle project](#import-gradle-project). If not, continue with manual configuration below... +If you see these, congrats! Skip to [building](#building-ghidra) or [developing](#developing-ghidra). If not, continue with manual download +instructions below... + +### Manual Download Instructions Create `~/.gradle/init.d/repos.gradle` with the following contents: @@ -118,7 +132,7 @@ mkdir ~/flatRepo If you prefer not to modify your user-wide Gradle configuration, you may use Gradle's other init script facilities, but you're on your own. -## Get Dependencies for FileFormats: +#### Get Dependencies for FileFormats: Download `dex-tools-2.0.zip` from the dex2jar project's releases page on GitHub. Unpack the `dex-*.jar` files from the `lib` directory to `~/flatRepo`: @@ -139,7 +153,7 @@ cd ~/flatRepo curl -OL https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar ``` -## Get Dependencies for DMG: +#### Get Dependencies for DMG: Download `hfsexplorer-0_21-bin.zip` from www.catacombae.org. Unpack the `lib` directory to `~/flatRepo.`: @@ -154,11 +168,77 @@ cd lib cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/flatRepo/ ``` -## Import Gradle Project +#### Get Dependencies for GhidraServer -If you want just to build Ghidra, you may skip ahead to Building Ghidra. -Otherwise, import Ghidra into Eclipse using the integrated BuildShip plugin. -Select File -> Import, expand Gradle, and select "Existing Gradle Project." +Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. +Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in a directory named: +`ghidra.bin/Ghidra/Features/GhidraServer/`. Note that `ghidra.bin` must be a sibling of `ghidra`: + +```bash +cd ~/Downloads # Or wherever +curl -OL https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip +mkdir -p ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ +cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ +``` + +#### Get Dependencies for GhidraDev + +Building the GhidraDev plugin for Eclipse requires the CDT and PyDev plugins for Eclipse. +Download `cdt-8.6.0.zip` from The Eclipse Foundation, and place it in a directory named: +`ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/`. Note that +`ghidra.bin` must be a sibling of `ghidra`. + +```bash +cd ~/Downloads # Or wherever +curl -OL 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' +curl -o 'cdt-8.6.0.zip.sha512' -L --retry 3 'http://www.eclipse.org/downloads/sums.php?type=sha512&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' +sha512sum -c 'cdt-8.6.0.zip.sha512' +mkdir -p ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ +cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ +``` + +Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same directory: + +```bash +cd ~/Downloads # Or wherever +curl -OL https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip +cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ +``` + +## Building Ghidra + +Before building, you may want to update the version and release name. +These properties are kept in `Ghidra/application.properties`. + +If you want it included, you must also build the GhidraDevPlugin module first. +Some supporting data will also be missing. +See the sections below for instructions to produce these components. +You may also be able to copy some of this data from a previous official distribution. + +To build the full package, use Gradle: + +```bash +gradle buildGhidra +``` + +The output will be placed in `build/dist/`. +It will be named according to the version, release name, build date, and platform. +To test it, unzip it where you like, and execute `./ghidraRun`. + +## Developing Ghidra + +### Prepare the Environment + +From the project root, execute: + +```bash +gradle prepDev +``` +The `prepDev` tasks primarily include generating some source, indexing our built-in help, and unpacking some dependencies. + +### Import Eclipse Projects +To develop/modify Ghidra, import Ghidra into Eclipse using the integrated BuildShip plugin. +Select __File -> Import__, expand Gradle, and select "Existing Gradle Project." Select the root of the source repo as the root Gradle project. Be sure to select Gradle 5.0, or point it at your local installation. You may see build path errors until the environment is properly prepared, as described below. @@ -171,32 +251,11 @@ From the project root: gradle eclipse ``` -Select File -> Import, expand General, and select "Existing Projects into Workspace." +Select __File -> Import__, expand General, and select "Existing Projects into Workspace." Select the root of the source repo, and select "Search for nested projects." Select all, and Finish. You may see build path errors until the environment is properly prepared, as described below. -## Prepare the Environment - -From the project root, execute: - -```bash -gradle prepDev -x yajswDevUnpack eclipse -``` -The `prepDev` tasks primarily include generating some source, indexing our built-in help, and unpacking some dependencies. -Regarding `yajswDevUnpack`, please see the relevant sections on GhidraServer below. -For now, we exclude the unpack task. - -Optionally, to pre-compile all the language modules, you may also execute: - -```bash -gradle sleighCompile -``` - -Refresh the projects in Eclipse. -You should not see any errors at this point, and you can accomplish many development tasks. -However, some features of Ghidra will not be functional until further steps are taken. - ### Building the natives Some of Ghidra's components are built for the native platform. @@ -226,68 +285,46 @@ gradle buildNatives_win64 This will build the decompiler, the demangler for GNU toolchains, the sleigh compiler, and (on Windows only) the PDB parser. -## Run Ghidra from Eclipse +### Pre-compile Language Modules (optional) + +Optionally, to pre-compile all the language modules, you may also execute: + +```bash +gradle sleighCompile +``` + +If the language modules are not pre-compiled, Ghidra will compile them at run time on an as-needed basis. + +### Import GhidraDev project (optional) + +Developing the GhidraDev Eclipse plugin requires the _Eclipse PDE (Plug-in Development Environment)_, which +can be installed via the Eclipse marketplace. It is also included in the _Eclipse IDE for RCP and RAP Developers_. +To generate the GhidraDev Eclipse projects, execute: + +``` +gradle eclipse -PeclipsePDE +``` + +Import the newly generated GhidraDev projects into Eclipse. + +__Note:__ If you are getting compilation errors related to PyDev and CDT, go into Eclipse's preferences, +and under _Target Platform_, activate _/Eclipse GhidraDevPlugin/GhidraDev.target_. + +### Run/Debug Ghidra from Eclipse To run or debug Ghidra from Eclipse, use the provided launch configuration (usually under the "Run" or "Debug" buttons). If the launcher does not appear, it probably has not been marked as a favorite. Click the dropdown next to the "Run" button and select "Run Configurations." Then expand "Java Application" on the left to find the "Ghidra" launcher. +## Building Supporting Data -# Building Ghidra - -To build the full Ghidra distribution, you must also build the GhidraServer. - -## Get Dependencies for GhidraServer - -_Note_: If you already ran the [dependency setup script](#dependency-setup-script), you can skip this section and continue on to [building the package](#building-the-package). - -Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. -Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in a directory named: -`ghidra.bin/Ghidra/Features/GhidraServer/`. Note that `ghidra.bin` must be a sibling of `ghidra`: - -```bash -cd ~/Downloads # Or wherever -curl -OL https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip -mkdir -p ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ -cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ -``` - -Use Gradle to unpack the wrapper for development. -From your clone: - -```bash -gradle yajswDevUnpack -``` - -## Building the Package - -Before building, you may want to update the version and release name. -These properties are kept in `Ghidra/application.properties`. - -If you want it included, you must also build the GhidraDevPlugin module first. -Some supporting data will also be missing. -See the sections below for instructions to produce these components. -You may also be able to copy some of this data from a previous official distribution. - -To build the full package, use Gradle: - -```bash -gradle buildGhidra -``` - -The output will be placed in `build/dist/`. -It will be named according to the version, release name, build date, and platform. -To test it, unzip it where you like, and execute `./ghidraRun`. - -# Building Supporting Data - -Some features of Ghidra require the curation of rather extensive data bases. +Some features of Ghidra require the curation of rather extensive databases. These include the Data Type Archives and Function ID Databases, both of which require collecting header files and libraries for the relevant SDKs and platforms. Much of this work is done by hand. The archives included in our official builds can be found in the [ghidra-data] repository. -## Building Data Type Archives +### Building Data Type Archives This task is often done manually from the Ghidra GUI, and the archives included in our official build require a fair bit of fine tuning. From a CodeBrowser window, select File -> Parse C Source. @@ -295,7 +332,7 @@ From here you can create and configure parsing profiles, which lists headers and Then, click "Parse to File" to create the Data Type Archive. The result can be added to an installation or source tree by copying it to `Ghidra/Features/Base/data/typeinfo`. -## Building FID Databases +### Building FID Databases This task is often done manually from the Ghidra GUI, and the archives included in our official build require a fair bit of fine tuning. You will first need to import the relevant libraries from which you'd like to produce a FID database. @@ -310,54 +347,3 @@ Now, select Tools -> Function ID -> Populate FidDb from programs. Fill out the options appropriately and click OK. If you'd like some details of our fine tuning, take a look at `Ghidra/Features/FunctionID/building_fid.txt`. - -# Developing / Building the GhidraDev Plugin - -First, install the Eclipse Plugin Development Environment (PDE). -By default, the GhidraDev project is excluded from the build. -To enable it, uncomment it in `settings.gradle`. - -```bash -${EDITOR:-vi} settings.gradle -``` - -You will need some additional runtime dependencies: - -## Get Dependencies for GhidraDev - -Building the GhidraDev plugin for Eclipse requires the CDT and PyDev plugins for Eclipse. -Download `cdt-8.6.0.zip` from The Eclipse Foundation, and place it in a directory named: -`ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/`. Note that -`ghidra.bin` must be a sibling of `ghidra`. - -```bash -cd ~/Downloads # Or wherever -curl -OL 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' -curl -o 'cdt-8.6.0.zip.sha512' -L --retry 3 'http://www.eclipse.org/downloads/sums.php?type=sha512&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' -sha512sum -c 'cdt-8.6.0.zip.sha512' -mkdir -p ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ -cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ -``` - -Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same directory: - -```bash -cd ~/Downloads # Or wherever -curl -OL https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip -cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ -``` - -Use Gradle to unpack the dependencies. -Note that these tasks will not work until you enable the GhidraDev project in `settings.gradle`. -From your clone: - -```bash -gradle cdtUnpack pyDevUnpack -``` - -## Import the GhidraDev Project - -If you're using BuildShip, simply refresh the Gradle project in Eclipse. -If you're not using BuildShip, re-run `gradle eclipse` and import the new project. - -[ghidra-data]: https://github.com/NationalSecurityAgency/ghidra-data From e326f98c0ec0c767a4468277092b4e38679c6649 Mon Sep 17 00:00:00 2001 From: adamopolous Date: Tue, 25 Jun 2019 09:00:40 -0400 Subject: [PATCH 3/9] GT-2897: changed location of tmp dir; added missing info to dev guide --- DevGuide.md | 4 +- gradle/init.gradle | 98 +++++++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/DevGuide.md b/DevGuide.md index f05b24525c..3b1f5e5374 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -80,7 +80,7 @@ manually-downloaded dependencies. The flat directory-style repository can be created and populated automatically by a provided script, or manually by downloading the required dependencies. Choose one of the two following methods: * [Automatic script instructions](#automatic-script-instructions) - * [Manual download instructions](#manual-download_instructions) + * [Manual download instructions](#manual-download-instructions) ### Automatic Script Instructions The flat directory-style repository can be setup automatically by running a simple Gradle script. @@ -101,6 +101,8 @@ the `init.gradle` script. If it ran correctly you will have a new folder, `flatR * hfsx * hfsx_dmglib * iharder-base64 + * cdt-8.6.0.zip + * PyDev 6.3.1.zip There will also be a new archive, yajsw-stable-12.12.zip, placed in __ghidra.bin/Ghidra/Features/GhidraServer/__. diff --git a/gradle/init.gradle b/gradle/init.gradle index adc52ecd78..bc98d87960 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -16,6 +16,8 @@ * - AXMLPrinter2.jar * * - hfsexplorer-0_21-bin.zip * * - yajsw-stable-12.12.zip (placed in GhidraServer location) * + * - cdt-8.6.0.zip * + * - PyDev 6.3.1.zip * * * * 2. Creates a gradle configuration file (repos.config) in * * /.gradle/init.d/. This contains repository * @@ -45,16 +47,19 @@ import org.apache.commons.io.*; import org.apache.commons.io.filefilter.*; ext.HOME_DIR = System.getProperty('user.home') -ext.FLAT_REPO_DIR = new File(HOME_DIR + "/flatRepo") -ext.TMP_FILE = File.createTempFile("downloads", ".tmp", null) -File TMP_DIR = TMP_FILE.getParentFile() +ext.FLAT_REPO_DIR = new File(HOME_DIR, "flatRepo") +File TMP_DIR = new File(System.getProperty('java.io.tmpdir')) ext.DOWNLOADS_DIR = new File(TMP_DIR, "ghidra") +ext.FILE_SIZE = 0; + // The URLs for each of the archives to be downloaded ext.DEX_ZIP = 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip' ext.AXML_ZIP = 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar' ext.HFS_ZIP = 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip' ext.YAJSW_ZIP = 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip' +ext.PYDEV_ZIP = 'https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip' +ext.CDT_ZIP = 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' // Store the MD5s for each of the downloads so we can verify that we retrieved them // all successfully @@ -62,11 +67,15 @@ ext.DEX_MD5 = '032456b9db9e6059376611553aecf31f' ext.AXML_MD5 = '55d70be9862c2b456cc91a933c197934' ext.HFS_MD5 = 'cc1713d634d2cd1fd7f21e18ae4d5d5c' ext.YAJSW_MD5 = 'e490ea92554f0238d74d4ef6161cb2c7' +ext.PYDEV_MD5 = '06263bdef4917c49d8d977d12c2d5073' +ext.CDT_MD5 = 'd41d8cd98f00b204e9800998ecf8427e' // Number of times to try and establish a connection when downloading files before // failing -ext.NUM_RETRIES = 1 +ext.NUM_RETRIES = 2 +// Set up a maven repository configuration so we can get access to Apache FileUtils for +// copying/deleting files. initscript { repositories { mavenCentral() @@ -76,6 +85,7 @@ initscript { } } +// This is where the real flow of the script starts... try { createDirs() createConfigFile() @@ -126,16 +136,20 @@ def createConfigFile() { * URL the attempt will be retried NUM_RETRIES times before failing. * * Progress is shown on the command line in the form of the number of bytes - * downloaded; the total size of the file to download is not known during the - * download so no percentage of the total is given. + * downloaded and a percentage of the total. + * + * Note: We do not validate that the number of bytes downloaded matches the + * expected total here; any discrepencies will be caught when checking + * the MD5s later on. * * @param url the file to download * @param filename the local file to create for the download */ def download(url, filename) { + println("File: " + url) BufferedInputStream istream = establishConnection(url, NUM_RETRIES); - assert istream != null : "***CONNECTION FAILURE***\nmax attempts exceeded; exiting\n" + assert istream != null : " ***CONNECTION FAILURE***\n max attempts exceeded; exiting\n" FileOutputStream ostream = new FileOutputStream(filename); def dataBuffer = new byte[1024]; @@ -147,8 +161,9 @@ def download(url, filename) { totalRead += bytesRead // print progress on the same line in the console... + int pctComplete = (totalRead / FILE_SIZE) * 100 print("\r") - print("Downloading: " + filename + " " + totalRead) + print(" Downloading: " + totalRead + " of " + FILE_SIZE + " (" + pctComplete + "%)") System.out.flush() } println("") @@ -158,21 +173,22 @@ def download(url, filename) { } /** - * Attemps to establish a connection to the given URL. This will attempt to retry the - * connection in the event of a failure. + * Attemps to establish a connection to the given URL. * * @param url the site to connect to * @param retries the number of times to attempt to reconnect if there is a failure + * @return the InputStream for the URL */ def establishConnection(url, retries) { - println("Download file: " + url) for (int i=0; i Date: Tue, 25 Jun 2019 11:37:13 -0400 Subject: [PATCH 4/9] GT-2897: User/script now puts yajsw.zip directly into the ghidra source repo eliminating the need for a ghidra.bin directory to be created. --- DevGuide.md | 4 +- Ghidra/Features/GhidraServer/build.gradle | 9 +++- gradle/init.gradle | 51 ++++------------------- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/DevGuide.md b/DevGuide.md index 3b1f5e5374..9adb647995 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -179,8 +179,8 @@ Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and ```bash cd ~/Downloads # Or wherever curl -OL https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip -mkdir -p ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ -cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra.bin/Ghidra/Features/GhidraServer/ +mkdir -p ~/git/ghidra/Ghidra/Features/GhidraServer/build/data/ +cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra/Ghidra/Features/GhidraServer/build/data/ ``` #### Get Dependencies for GhidraDev diff --git a/Ghidra/Features/GhidraServer/build.gradle b/Ghidra/Features/GhidraServer/build.gradle index 1d89b58e34..8a1a8ecb0e 100644 --- a/Ghidra/Features/GhidraServer/build.gradle +++ b/Ghidra/Features/GhidraServer/build.gradle @@ -25,7 +25,14 @@ addExports([ ]) CopySpec yajswCopySpec = copySpec { - from(zipTree("${BIN_REPO}/Ghidra/Features/GhidraServer/${yajswRelease}.zip")) { + File localFile = file("build/data/${yajswRelease}.zip") + File binFile = file("${BIN_REPO}/Ghidra/Features/GhidraServer/${yajswRelease}.zip") + + // First check if the file was downloaded and dropped in locally. If not, check in the bin + // repo. + def yajswZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + + from(yajswZipTree) { include "${yajswRelease}/lib/core/**" include "${yajswRelease}/lib/extended/**" include "${yajswRelease}/templates/**" diff --git a/gradle/init.gradle b/gradle/init.gradle index bc98d87960..1aa9d3549f 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -9,13 +9,13 @@ * * * Specifically, this task: * * * - * 1. Downloads various jars required by the ghidra build and * - * puts them in /flatRepo. The jars to be * + * 1. Downloads various dependencies required by the ghidra build and * + * puts them in /flatRepo. The files to be * * downloaded: * * - dex-tools-2.0.zip * * - AXMLPrinter2.jar * * - hfsexplorer-0_21-bin.zip * - * - yajsw-stable-12.12.zip (placed in GhidraServer location) * + * - yajsw-stable-12.12.zip * * - cdt-8.6.0.zip * * - PyDev 6.3.1.zip * * * @@ -47,6 +47,7 @@ import org.apache.commons.io.*; import org.apache.commons.io.filefilter.*; ext.HOME_DIR = System.getProperty('user.home') +ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile() ext.FLAT_REPO_DIR = new File(HOME_DIR, "flatRepo") File TMP_DIR = new File(System.getProperty('java.io.tmpdir')) ext.DOWNLOADS_DIR = new File(TMP_DIR, "ghidra") @@ -245,10 +246,9 @@ def populateFlatRepo() { // 2. Unzip the dependencies unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex-tools-2.0.zip") unzipHfsx() - unzipYajsw() - // 3. Copy the necessary jars to the flatRepo directory. Yajsw is the - // exception; it needs to go in a GhidraServer folder. + // 3. Copy the necessary jars to the flatRepo directory. Yajsw, CDT, and PyDev go directly into + // the source repository. copyDexTools() copyHfsx() copyYajsw() @@ -283,14 +283,6 @@ def unzipHfsx() { unzip (DOWNLOADS_DIR, hfsxdir, "hfsexplorer-0_21-bin.zip") } -/** - * Unzips the yajsw zip file - */ -def unzipYajsw() { - def yajswdir = getOrCreateTempYajswDir() - unzip (DOWNLOADS_DIR, yajswdir, "yajsw-stable-12.12.zip") -} - /** * Copies the dex-tools jars to the flat repository * @@ -312,36 +304,23 @@ def copyHfsx() { /** * Copies the yajswdir zip to its location in the GhidraServer project. - * - * Note: There is a Gradle task to do all of this (yajswDevUnpack) but we cannot execute - * that from here so just copy everything manually */ def copyYajsw() { - def userdir = System.getProperty("user.dir") - def yajswdir = new File(getOrCreateTempYajswDir(),"yajsw-stable-12.12") - def yajswdirTarget = new File(userdir, "/../ghidra/Ghidra/Features/GhidraServer/build/data/yajsw-stable-12.12") - - FileUtils.copyFile(new File(yajswdir, "yajsw.policy.txt"), new File(yajswdirTarget, "yajsw.policy.txt")); - FileUtils.copyFile(new File(yajswdir, "LICENSE.txt"), new File(yajswdirTarget, "LICENSE.txt")); - FileUtils.copyDirectory(yajswdir, yajswdirTarget, new WildcardFileFilter("*.jar")); - FileUtils.copyDirectory(new File(yajswdir, "templates"), new File(yajswdirTarget, "templates")); - FileUtils.copyDirectory(new File(yajswdir, "lib/extended"), new File(yajswdirTarget, "lib/extended")); - FileUtils.copyDirectory(new File(yajswdir, "doc"), new File(yajswdirTarget, "doc")); - FileUtils.copyDirectory(new File(yajswdir, "lib/core"), new File(yajswdirTarget, "lib/core")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, "yajsw-stable-12.12.zip"), new File(REPO_DIR, "Ghidra/Features/GhidraServer/build/data/yajsw-stable-12.12.zip")); } /** * Copies the pydev zip to its bin repository location */ def copyPyDev() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, '/PyDev 6.3.1.zip'), new File(HOME_DIR, '/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/PyDev 6.3.1.zip')); + FileUtils.copyFile(new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/data/buildDependencies/PyDev 6.3.1.zip")); } /** * Copies the cdt zip to its bin repository location */ def copyCdt() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, '/cdt-8.6.0.zip'), new File(HOME_DIR, '/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/cdt-8.6.0.zip')); + FileUtils.copyFile(new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/data/buildDependencies/cdt-8.6.0.zip")); } /** @@ -358,18 +337,6 @@ def getOrCreateTempHfsxDir() { return hfsxdir; } -/** - * Creates a temporary folder to house the yajsw zip contents - */ -def getOrCreateTempYajswDir() { - def yajswdir = new File (DOWNLOADS_DIR, "yajsw") - if (!yajswdir.exists()) { - yajswdir.mkdir() - } - - return yajswdir; -} - /** * Performs any cleanup operations that need to be performed after the flat repo has * been populated. From f866b0f5e7e23bee2f7d8ba499d24cc010286182 Mon Sep 17 00:00:00 2001 From: adamopolous Date: Tue, 25 Jun 2019 13:28:34 -0400 Subject: [PATCH 5/9] GT-2897: moved downloads to ghidra/build --- gradle/init.gradle | 125 +++++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 37 deletions(-) diff --git a/gradle/init.gradle b/gradle/init.gradle index 1aa9d3549f..0b6898cf3c 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -49,8 +49,7 @@ import org.apache.commons.io.filefilter.*; ext.HOME_DIR = System.getProperty('user.home') ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile() ext.FLAT_REPO_DIR = new File(HOME_DIR, "flatRepo") -File TMP_DIR = new File(System.getProperty('java.io.tmpdir')) -ext.DOWNLOADS_DIR = new File(TMP_DIR, "ghidra") +ext.DOWNLOADS_DIR = new File(REPO_DIR, "build/downloads") ext.FILE_SIZE = 0; @@ -162,9 +161,14 @@ def download(url, filename) { totalRead += bytesRead // print progress on the same line in the console... - int pctComplete = (totalRead / FILE_SIZE) * 100 print("\r") - print(" Downloading: " + totalRead + " of " + FILE_SIZE + " (" + pctComplete + "%)") + if (FILE_SIZE.equals("unknown")) { + print(" Downloading: " + totalRead + " of " + FILE_SIZE) + } + else { + int pctComplete = (totalRead / FILE_SIZE) * 100 + print(" Downloading: " + totalRead + " of " + FILE_SIZE + " (" + pctComplete + "%)") + } System.out.flush() } println("") @@ -186,6 +190,11 @@ def establishConnection(url, retries) { println(" Connect attempt " + (i+1) + " of " + retries) URLConnection conn = new URL(url).openConnection(); FILE_SIZE = conn.getContentLength(); + if (FILE_SIZE == -1) { + // This can happen if there is a problem retrieving the size; we've seen it happen + // in testing. + FILE_SIZE = "unknown" + } return new BufferedInputStream(new URL(url).openStream()); } catch (Exception e) { @@ -219,30 +228,57 @@ def unzip(sourceDir, targetDir, zipFileName) { } /** - * Downloads and stores the necessary dependencies in the local - * flat repository. + * Downloads and stores the necessary dependencies in the local flat repository. + * + * If the dependency already exists in the downloads folder (DOWNLOADS_DIR) and has the + * proper checksum, it will NOT be re-downloaded. */ def populateFlatRepo() { - // 1. Download all the dependencies. - download (DEX_ZIP, DOWNLOADS_DIR.path + '/dex-tools-2.0.zip') - validateChecksum(DOWNLOADS_DIR.path + '/dex-tools-2.0.zip', DEX_MD5); + // 1. Download all the dependencies and verify their checksums. If the dependency has already + // been download, do NOT download again. + File file = new File(DOWNLOADS_DIR.path, '/dex-tools-2.0.zip') + def checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(DEX_MD5)))) { + download (DEX_ZIP, file.path) + validateChecksum(checksum, DEX_MD5); + } + + file = new File(FLAT_REPO_DIR.path + '/AXMLPrinter2.jar') + checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(AXML_MD5)))) { + download (AXML_ZIP, file.path) + validateChecksum(checksum, AXML_MD5); + } - download (AXML_ZIP, FLAT_REPO_DIR.path + '/AXMLPrinter2.jar') - validateChecksum(FLAT_REPO_DIR.path + '/AXMLPrinter2.jar', AXML_MD5); + file = new File(DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip') + checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(HFS_MD5)))) { + download (HFS_ZIP, file.path) + validateChecksum(checksum, HFS_MD5); + } - download (HFS_ZIP, DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip') - validateChecksum(DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip', HFS_MD5); + file = new File(DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip') + checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(YAJSW_MD5)))) { + download (YAJSW_ZIP, file.path) + validateChecksum(checksum, YAJSW_MD5); + } - download (YAJSW_ZIP, DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip') - validateChecksum(DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip', YAJSW_MD5); + file = new File(DOWNLOADS_DIR.path + "/PyDev 6.3.1.zip") + checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(PYDEV_MD5)))) { + download (PYDEV_ZIP, file.path) + validateChecksum(checksum, PYDEV_MD5); + } - download (PYDEV_ZIP, DOWNLOADS_DIR.path + '/PyDev 6.3.1.zip') - validateChecksum(DOWNLOADS_DIR.path + '/PyDev 6.3.1.zip', PYDEV_MD5); + file = new File(DOWNLOADS_DIR.path + '/cdt-8.6.0.zip') + checksum = generateChecksum(file) + if (!(file.exists() && (checksum.equals(CDT_MD5)))) { + download (CDT_ZIP, file.path) + validateChecksum(checksum, CDT_MD5); + } - download (CDT_ZIP, DOWNLOADS_DIR.path + '/cdt-8.6.0.zip') - validateChecksum(DOWNLOADS_DIR.path + '/cdt-8.6.0.zip', CDT_ZIP); - // 2. Unzip the dependencies unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex-tools-2.0.zip") unzipHfsx() @@ -257,22 +293,34 @@ def populateFlatRepo() { } /** - * Generates the md5 for the given file and compares it against the - * expected result. If there is no match an assert exception will be - * generated. + * Generates the md5 for the given file * - * @param filename the fully-qualified file path+name - * @param expectedMd5 the expected md5 for the file + * @param file the file to generate the checksum for + * @return the generated checksum */ -def validateChecksum(filename, expectedMd5) { - MessageDigest md = MessageDigest.getInstance("MD5"); - md.update(Files.readAllBytes(Paths.get(filename))); - byte[] digest = md.digest(); - StringBuilder sb = new StringBuilder(); - for (byte b : digest) { - sb.append(String.format("%02x", b)); - } - assert(sb.toString().equals(expectedMd5)); +def generateChecksum(file) { + if (!file.exists()) { + return + } + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(Paths.get(file.path))); + byte[] digest = md.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + + return sb.toString(); +} + +/** + * Compares two checksums and generates an assert failure if they do not match + * + * @param sourceMd5 the checksum to validate + * @param expectedMd5 the expected checksum + */ +def validateChecksum(sourceMd5, expectedMd5) { + assert(sourceMd5.equals(expectedMd5)); } /** @@ -342,7 +390,10 @@ def getOrCreateTempHfsxDir() { * been populated. */ def cleanup() { - if (DOWNLOADS_DIR.exists()) { - FileUtils.deleteDirectory(DOWNLOADS_DIR) - } + // Uncomment this if we want to delete the downloads folder. For now, leave this and + // depend on a gradle clean to wipe it out. + // + //if (DOWNLOADS_DIR.exists()) { + // FileUtils.deleteDirectory(DOWNLOADS_DIR) + //} } \ No newline at end of file From 5f388e6bbc1c48bf0748fc4307baa3d58536d491 Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Tue, 25 Jun 2019 15:20:25 -0400 Subject: [PATCH 6/9] GT-2897: More improvements. --- DevGuide.md | 21 +++--- Ghidra/Features/GhidraServer/build.gradle | 2 +- .../GhidraDev/GhidraDevPlugin/build.gradle | 34 +++++---- gradle/init.gradle | 70 ++++++++++--------- 4 files changed, 69 insertions(+), 58 deletions(-) diff --git a/DevGuide.md b/DevGuide.md index 9adb647995..c3c93c20a6 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -173,30 +173,29 @@ cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/flatRepo/ #### Get Dependencies for GhidraServer Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. -Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in a directory named: -`ghidra.bin/Ghidra/Features/GhidraServer/`. Note that `ghidra.bin` must be a sibling of `ghidra`: +Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in: +`~/ghidra/Ghidra/Features/GhidraServer/build`: ```bash cd ~/Downloads # Or wherever curl -OL https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip -mkdir -p ~/git/ghidra/Ghidra/Features/GhidraServer/build/data/ -cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra/Ghidra/Features/GhidraServer/build/data/ +mkdir -p ~/git/ghidra/Ghidra/Features/GhidraServer/build/ +cp ~/Downloads/yajsw-stable-12.12.zip ~/git/ghidra/Ghidra/Features/GhidraServer/build/ ``` #### Get Dependencies for GhidraDev Building the GhidraDev plugin for Eclipse requires the CDT and PyDev plugins for Eclipse. -Download `cdt-8.6.0.zip` from The Eclipse Foundation, and place it in a directory named: -`ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/`. Note that -`ghidra.bin` must be a sibling of `ghidra`. +Download `cdt-8.6.0.zip` from The Eclipse Foundation, and place it in: +`~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`: ```bash cd ~/Downloads # Or wherever curl -OL 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' curl -o 'cdt-8.6.0.zip.sha512' -L --retry 3 'http://www.eclipse.org/downloads/sums.php?type=sha512&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' -sha512sum -c 'cdt-8.6.0.zip.sha512' -mkdir -p ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ -cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ +shasum -a 512 -c 'cdt-8.6.0.zip.sha512' +mkdir -p ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ +cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ ``` Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same directory: @@ -204,7 +203,7 @@ Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same director ```bash cd ~/Downloads # Or wherever curl -OL https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip -cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/ +cp ~/Downloads/'PyDev 6.3.1.zip ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ ``` ## Building Ghidra diff --git a/Ghidra/Features/GhidraServer/build.gradle b/Ghidra/Features/GhidraServer/build.gradle index 8a1a8ecb0e..1f5a16c73e 100644 --- a/Ghidra/Features/GhidraServer/build.gradle +++ b/Ghidra/Features/GhidraServer/build.gradle @@ -25,7 +25,7 @@ addExports([ ]) CopySpec yajswCopySpec = copySpec { - File localFile = file("build/data/${yajswRelease}.zip") + File localFile = file("build/${yajswRelease}.zip") File binFile = file("${BIN_REPO}/Ghidra/Features/GhidraServer/${yajswRelease}.zip") // First check if the file was downloaded and dropped in locally. If not, check in the bin diff --git a/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle b/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle index a8fce00096..b2b80562ff 100644 --- a/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle +++ b/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build.gradle @@ -42,22 +42,14 @@ dependencies { compileJava.enabled = false jar.enabled = false -File libraryJarDestDir = file("build/data") - -File pyDevSourceZipFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/PyDev 6.3.1.zip") -File cdtSourceZipFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/cdt-8.6.0.zip") - -File pyDevDestDir = file("build/data/buildDependencies/pydev") -File cdtDestDir = file("build/data/buildDependencies/cdt") - task utilityJar(type:Copy) { - destinationDir libraryJarDestDir + destinationDir file("build/data") from { project(':Utility').jar } // using closure to delay until all projects evaluated } task launchSupportJar(type:Copy) { - destinationDir libraryJarDestDir + destinationDir file("build/data") from { project(':LaunchSupport').jar } // using closure to delay until all projects evaluated } @@ -65,13 +57,22 @@ task launchSupportJar(type:Copy) { task pyDevUnpack(type:Copy) { description "Unpack PyDev plugin archive for development use" group "Development Preparation" + + File pyDevDestDir = file("build/data/buildDependencies/pydev") // Without this, the copyTask will unzip the file to check for "up to date" onlyIf { !pyDevDestDir.exists() } + + File localFile = file("build/PyDev 6.3.1.zip") + File binFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/PyDev 6.3.1.zip") - from zipTree(pyDevSourceZipFile) + // First check if the file was downloaded and dropped in locally. If not, check in the bin + // repo. + def pyDevZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + + from pyDevZipTree exclude "**/.project", "**/.pydevproject" destinationDir pyDevDestDir @@ -80,13 +81,22 @@ task pyDevUnpack(type:Copy) { task cdtUnpack(type:Copy) { description "Unpack CDT plugin archive for development use" group "Development Preparation" + + File cdtDestDir = file("build/data/buildDependencies/cdt") // Without this, the copyTask will unzip the file to check for "up to date" onlyIf { !cdtDestDir.exists() } + + File localFile = file("build/cdt-8.6.0.zip") + File binFile = file("${BIN_REPO}/GhidraBuild/EclipsePlugins/GhidraDev/buildDependencies/cdt-8.6.0.zip") - from zipTree(cdtSourceZipFile) + // First check if the file was downloaded and dropped in locally. If not, check in the bin + // repo. + def cdtZipTree = localFile.exists() ? zipTree(localFile) : zipTree(binFile) + + from cdtZipTree destinationDir cdtDestDir } diff --git a/gradle/init.gradle b/gradle/init.gradle index 0b6898cf3c..50a3674a28 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -68,7 +68,7 @@ ext.AXML_MD5 = '55d70be9862c2b456cc91a933c197934' ext.HFS_MD5 = 'cc1713d634d2cd1fd7f21e18ae4d5d5c' ext.YAJSW_MD5 = 'e490ea92554f0238d74d4ef6161cb2c7' ext.PYDEV_MD5 = '06263bdef4917c49d8d977d12c2d5073' -ext.CDT_MD5 = 'd41d8cd98f00b204e9800998ecf8427e' +ext.CDT_MD5 = '8e9438a6e3947d614af98e1b58e945a2' // Number of times to try and establish a connection when downloading files before // failing @@ -123,11 +123,11 @@ def createConfigFile() { repoConfigFile.write("ext.HOME = System.getProperty('user.home')") repoConfigFile.append("\nallprojects {") - repoConfigFile.append("\nrepositories {") - repoConfigFile.append("\nmavenCentral()") - repoConfigFile.append("\njcenter()") - repoConfigFile.append('\nflatDir name: "flat", dirs:["$HOME/flatRepo"]') - repoConfigFile.append("\n}") + repoConfigFile.append("\n\trepositories {") + repoConfigFile.append("\n\t\tmavenCentral()") + repoConfigFile.append("\n\t\tjcenter()") + repoConfigFile.append('\n\t\tflatDir name: "flat", dirs:["$HOME/flatRepo"]') + repoConfigFile.append("\n\t}") repoConfigFile.append("\n}") } @@ -237,46 +237,40 @@ def populateFlatRepo() { // 1. Download all the dependencies and verify their checksums. If the dependency has already // been download, do NOT download again. - File file = new File(DOWNLOADS_DIR.path, '/dex-tools-2.0.zip') - def checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(DEX_MD5)))) { + File file = new File(DOWNLOADS_DIR, 'dex-tools-2.0.zip') + if (!DEX_MD5.equals(generateChecksum(file))) { download (DEX_ZIP, file.path) - validateChecksum(checksum, DEX_MD5); + validateChecksum(generateChecksum(file), DEX_MD5); } - file = new File(FLAT_REPO_DIR.path + '/AXMLPrinter2.jar') - checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(AXML_MD5)))) { + file = new File(DOWNLOADS_DIR, 'AXMLPrinter2.jar') + if (!AXML_MD5.equals(generateChecksum(file))) { download (AXML_ZIP, file.path) - validateChecksum(checksum, AXML_MD5); + validateChecksum(generateChecksum(file), AXML_MD5); } - file = new File(DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip') - checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(HFS_MD5)))) { + file = new File(DOWNLOADS_DIR, 'hfsexplorer-0_21-bin.zip') + if (!HFS_MD5.equals(generateChecksum(file))) { download (HFS_ZIP, file.path) - validateChecksum(checksum, HFS_MD5); + validateChecksum(generateChecksum(file), HFS_MD5); } - file = new File(DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip') - checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(YAJSW_MD5)))) { + file = new File(DOWNLOADS_DIR, 'yajsw-stable-12.12.zip') + if (!YAJSW_MD5.equals(generateChecksum(file))) { download (YAJSW_ZIP, file.path) - validateChecksum(checksum, YAJSW_MD5); + validateChecksum(generateChecksum(file), YAJSW_MD5); } - file = new File(DOWNLOADS_DIR.path + "/PyDev 6.3.1.zip") - checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(PYDEV_MD5)))) { + file = new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip') + if (!PYDEV_MD5.equals(generateChecksum(file))) { download (PYDEV_ZIP, file.path) - validateChecksum(checksum, PYDEV_MD5); + validateChecksum(generateChecksum(file), PYDEV_MD5); } - file = new File(DOWNLOADS_DIR.path + '/cdt-8.6.0.zip') - checksum = generateChecksum(file) - if (!(file.exists() && (checksum.equals(CDT_MD5)))) { + file = new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip') + if (!CDT_MD5.equals(generateChecksum(file))) { download (CDT_ZIP, file.path) - validateChecksum(checksum, CDT_MD5); + validateChecksum(generateChecksum(file), CDT_MD5); } // 2. Unzip the dependencies @@ -286,6 +280,7 @@ def populateFlatRepo() { // 3. Copy the necessary jars to the flatRepo directory. Yajsw, CDT, and PyDev go directly into // the source repository. copyDexTools() + copyAXML() copyHfsx() copyYajsw() copyPyDev() @@ -300,7 +295,7 @@ def populateFlatRepo() { */ def generateChecksum(file) { if (!file.exists()) { - return + return null } MessageDigest md = MessageDigest.getInstance("MD5"); md.update(Files.readAllBytes(Paths.get(file.path))); @@ -340,6 +335,13 @@ def copyDexTools() { FileUtils.copyDirectory(new File(DOWNLOADS_DIR, 'dex2jar-2.0/lib/'), FLAT_REPO_DIR, new WildcardFileFilter("dex-*")); } +/** + * Copies the AXMLPrinter2 jar to the flat repository + */ +def copyAXML() { + FileUtils.copyFile(new File(DOWNLOADS_DIR, 'AXMLPrinter2.jar'), new File(FLAT_REPO_DIR, "AXMLPrinter2.jar")); +} + /** * Copies the necessary hfsx jars to the flat repository */ @@ -354,21 +356,21 @@ def copyHfsx() { * Copies the yajswdir zip to its location in the GhidraServer project. */ def copyYajsw() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, "yajsw-stable-12.12.zip"), new File(REPO_DIR, "Ghidra/Features/GhidraServer/build/data/yajsw-stable-12.12.zip")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, "yajsw-stable-12.12.zip"), new File(REPO_DIR, "Ghidra/Features/GhidraServer/build/yajsw-stable-12.12.zip")); } /** * Copies the pydev zip to its bin repository location */ def copyPyDev() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/data/buildDependencies/PyDev 6.3.1.zip")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, 'PyDev 6.3.1.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/PyDev 6.3.1.zip")); } /** * Copies the cdt zip to its bin repository location */ def copyCdt() { - FileUtils.copyFile(new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/data/buildDependencies/cdt-8.6.0.zip")); + FileUtils.copyFile(new File(DOWNLOADS_DIR, 'cdt-8.6.0.zip'), new File(REPO_DIR, "GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/cdt-8.6.0.zip")); } /** From 5e1ea55a6aac999a9e61f866a0869e393c57fc45 Mon Sep 17 00:00:00 2001 From: adamopolous Date: Wed, 26 Jun 2019 06:48:52 -0400 Subject: [PATCH 7/9] GT-2897: fixed some documentation issues --- DevGuide.md | 2 +- gradle/init.gradle | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/DevGuide.md b/DevGuide.md index c3c93c20a6..8e3be3668f 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -104,7 +104,7 @@ the `init.gradle` script. If it ran correctly you will have a new folder, `flatR * cdt-8.6.0.zip * PyDev 6.3.1.zip -There will also be a new archive, yajsw-stable-12.12.zip, placed in __ghidra.bin/Ghidra/Features/GhidraServer/__. +There will also be a new archive, yajsw-stable-12.12.zip, placed in `ghidra.bin/Ghidra/Features/GhidraServer/`. If you see these, congrats! Skip to [building](#building-ghidra) or [developing](#developing-ghidra). If not, continue with manual download instructions below... diff --git a/gradle/init.gradle b/gradle/init.gradle index 50a3674a28..68ea8d3950 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -10,7 +10,8 @@ * Specifically, this task: * * * * 1. Downloads various dependencies required by the ghidra build and * - * puts them in /flatRepo. The files to be * + * puts them in /ghidra/downloads/. From here they are * + * unzipped and/or copied to their final locations. The files to be * * downloaded: * * - dex-tools-2.0.zip * * - AXMLPrinter2.jar * @@ -29,14 +30,8 @@ * * * gradle --init-script gradle/init.gradle * * * - * Note: Running this script multiple times will cause the config * - * file to be recreated and all dependencies re-downloaded. * - * * - * Note: All files are downloaded to a the standard java temporary folder * - * location, in a sub-folder called 'ghidra. This is cleaned up and * - * removed when the script completes. * - * TODO: make sure this folder is cleaned up in EVERY case, especially * - * if the script fails at some point * + * Note: When running the script, files will only be downloaded if * + * necessary (eg: they are not already in the downloads/ folder). * * * * *******************************************************************************/ @@ -51,9 +46,10 @@ ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getPar ext.FLAT_REPO_DIR = new File(HOME_DIR, "flatRepo") ext.DOWNLOADS_DIR = new File(REPO_DIR, "build/downloads") +// Stores the size of the file being downloaded (for formatting print statements) ext.FILE_SIZE = 0; -// The URLs for each of the archives to be downloaded +// The URLs for each of the dependencies ext.DEX_ZIP = 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip' ext.AXML_ZIP = 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar' ext.HFS_ZIP = 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip' @@ -61,8 +57,7 @@ ext.YAJSW_ZIP = 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable ext.PYDEV_ZIP = 'https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip' ext.CDT_ZIP = 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' -// Store the MD5s for each of the downloads so we can verify that we retrieved them -// all successfully +// The MD5s for each of the dependencies ext.DEX_MD5 = '032456b9db9e6059376611553aecf31f' ext.AXML_MD5 = '55d70be9862c2b456cc91a933c197934' ext.HFS_MD5 = 'cc1713d634d2cd1fd7f21e18ae4d5d5c' @@ -160,7 +155,6 @@ def download(url, filename) { ostream.write(dataBuffer, 0, bytesRead); totalRead += bytesRead - // print progress on the same line in the console... print("\r") if (FILE_SIZE.equals("unknown")) { print(" Downloading: " + totalRead + " of " + FILE_SIZE) From b69e737a6033e8a2ba8ebd7e6658a3c18b4fa6c9 Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Wed, 26 Jun 2019 11:05:42 -0400 Subject: [PATCH 8/9] GT-2897: Moving flatRepo to installation dir. --- .gitignore | 1 + DevGuide.md | 90 ++++++++++++++++++++-------------------------- build.gradle | 13 +++++++ gradle/init.gradle | 27 +------------- 4 files changed, 54 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 90f4b9de6c..3e2c81f0cc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ghidra.repos.config # Misc files produced while executing application repositories/ +flatRepo/ Ghidra/.ghidraSvrKeys wrapper.log* diff --git a/DevGuide.md b/DevGuide.md index 8e3be3668f..760cc1843c 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -84,12 +84,13 @@ or manually by downloading the required dependencies. Choose one of the two fol ### Automatic Script Instructions The flat directory-style repository can be setup automatically by running a simple Gradle script. -Navigate to the Ghidra clone directory you just created, and run the following: +Navigate to `~/git/ghidra` and run the following: ``` gradle --init-script gradle/init.gradle tasks ``` The Gradle task to be executed, in this case _tasks_, is unimportant. The point is to have Gradle execute -the `init.gradle` script. If it ran correctly you will have a new folder, `flatRepo/`, in your home directory populated with the following jar files: +the `init.gradle` script. If it ran correctly you will have a new `~/git/ghidra/flatRepo/` +directory populated with the following jar files: * AXMLPrinter2 * csframework * dex-ir-2.0 @@ -100,65 +101,49 @@ the `init.gradle` script. If it ran correctly you will have a new folder, `flatR * dex-writer-2.0 * hfsx * hfsx_dmglib - * iharder-base64 - * cdt-8.6.0.zip - * PyDev 6.3.1.zip + * iharder-base64 -There will also be a new archive, yajsw-stable-12.12.zip, placed in `ghidra.bin/Ghidra/Features/GhidraServer/`. +There will also be a new archive files at: + * ~/git/ghidra/Ghidra/Features/GhidraServer/build/`yajsw-stable-12.12.zip` + * ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`PyDev 6.3.1.zip` + * ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/`cdt-8.6.0.zip` If you see these, congrats! Skip to [building](#building-ghidra) or [developing](#developing-ghidra). If not, continue with manual download instructions below... ### Manual Download Instructions -Create `~/.gradle/init.d/repos.gradle` with the following contents: - -```groovy -ext.HOME = System.getProperty('user.home') - -allprojects { - repositories { - mavenCentral() - jcenter() - flatDir name:'flat', dirs:["$HOME/flatRepo"] - } -} -``` - -Create the `~/flatRepo` folder to hold the manually-downloaded dependencies: +Create the `~/git/ghidra/flatRepo/` directory to hold the manually-downloaded dependencies: ```bash -mkdir ~/flatRepo +mkdir ~/git/ghidra/flatRepo ``` -If you prefer not to modify your user-wide Gradle configuration, you may use -Gradle's other init script facilities, but you're on your own. - #### Get Dependencies for FileFormats: Download `dex-tools-2.0.zip` from the dex2jar project's releases page on GitHub. -Unpack the `dex-*.jar` files from the `lib` directory to `~/flatRepo`: +Unpack the `dex-*.jar` files from the `lib` directory to `~/git/ghidra/flatRepo`: ```bash cd ~/Downloads # Or wherever curl -OL https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip unzip dex-tools-2.0.zip -cp dex2jar-2.0/lib/dex-*.jar ~/flatRepo/ +cp dex2jar-2.0/lib/dex-*.jar ~/git/ghidra/flatRepo/ ``` Download `AXMLPrinter2.jar` from the "android4me" archive on code.google.com. -Place it in `~/flatRepo`: +Place it in `~/git/ghidra/flatRepo`: ```bash -cd ~/flatRepo +cd ~/git/ghidra/flatRepo curl -OL https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar ``` #### Get Dependencies for DMG: Download `hfsexplorer-0_21-bin.zip` from www.catacombae.org. -Unpack the `lib` directory to `~/flatRepo.`: +Unpack the `lib` directory to `~/git/ghidra/flatRepo`: ```bash cd ~/Downloads # Or wherever @@ -167,14 +152,14 @@ mkdir hfsx cd hfsx unzip ../hfsexplorer-0_21-bin.zip cd lib -cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/flatRepo/ +cp csframework.jar hfsx_dmglib.jar hfsx.jar iharder-base64.jar ~/git/ghidra/flatRepo/ ``` #### Get Dependencies for GhidraServer Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12. Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in: -`~/ghidra/Ghidra/Features/GhidraServer/build`: +`~/git/ghidra/Ghidra/Features/GhidraServer/build`: ```bash cd ~/Downloads # Or wherever @@ -194,7 +179,7 @@ cd ~/Downloads # Or wherever curl -OL 'http://www.eclipse.org/downloads/download.php?r=1&protocol=https&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' curl -o 'cdt-8.6.0.zip.sha512' -L --retry 3 'http://www.eclipse.org/downloads/sums.php?type=sha512&file=/tools/cdt/releases/8.6/cdt-8.6.0.zip' shasum -a 512 -c 'cdt-8.6.0.zip.sha512' -mkdir -p ~/git/ghidra.bin/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ +mkdir -p ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ cp ~/Downloads/cdt-8.6.0.zip ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ ``` @@ -202,19 +187,14 @@ Download `PyDev 6.3.1.zip` from www.pydev.org, and place it in the same director ```bash cd ~/Downloads # Or wherever -curl -OL https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip -cp ~/Downloads/'PyDev 6.3.1.zip ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ +curl -L -o 'PyDev 6.3.1.zip' https://sourceforge.net/projects/pydev/files/pydev/PyDev%206.3.1/PyDev%206.3.1.zip +cp ~/Downloads/'PyDev 6.3.1.zip' ~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build/ ``` ## Building Ghidra Before building, you may want to update the version and release name. -These properties are kept in `Ghidra/application.properties`. - -If you want it included, you must also build the GhidraDevPlugin module first. -Some supporting data will also be missing. -See the sections below for instructions to produce these components. -You may also be able to copy some of this data from a previous official distribution. +These properties are kept in `~/git/ghidra/Ghidra/application.properties`. To build the full package, use Gradle: @@ -222,10 +202,15 @@ To build the full package, use Gradle: gradle buildGhidra ``` -The output will be placed in `build/dist/`. +The output will be placed in `~/git/ghidra/build/dist/`. It will be named according to the version, release name, build date, and platform. To test it, unzip it where you like, and execute `./ghidraRun`. +__NOTE:__ Unless pre-built manually, the Eclipse GhidraDev plugin will not be included +in the build. In addition, some other supporting data will also be missing. +See the sections below for instructions on how to produce these components. +You may also be able to copy some of these already-built components from a previous official distribution. + ## Developing Ghidra ### Prepare the Environment @@ -296,7 +281,7 @@ gradle sleighCompile If the language modules are not pre-compiled, Ghidra will compile them at run time on an as-needed basis. -### Import GhidraDev project (optional) +### Import and Build GhidraDev project (optional) Developing the GhidraDev Eclipse plugin requires the _Eclipse PDE (Plug-in Development Environment)_, which can be installed via the Eclipse marketplace. It is also included in the _Eclipse IDE for RCP and RAP Developers_. @@ -311,6 +296,9 @@ Import the newly generated GhidraDev projects into Eclipse. __Note:__ If you are getting compilation errors related to PyDev and CDT, go into Eclipse's preferences, and under _Target Platform_, activate _/Eclipse GhidraDevPlugin/GhidraDev.target_. +See `~/git/ghidra/GhidraBuild/EclipsePlugins/GhidraDev/GhidraDevPlugin/build_README.txt` +for instructions on how to build the GhidraDev plugin. + ### Run/Debug Ghidra from Eclipse To run or debug Ghidra from Eclipse, use the provided launch configuration (usually under the "Run" or "Debug" buttons). @@ -323,15 +311,15 @@ Then expand "Java Application" on the left to find the "Ghidra" launcher. Some features of Ghidra require the curation of rather extensive databases. These include the Data Type Archives and Function ID Databases, both of which require collecting header files and libraries for the relevant SDKs and platforms. Much of this work is done by hand. -The archives included in our official builds can be found in the [ghidra-data] repository. +The archives included in our official builds can be found in the __[ghidra-data]__ repository. ### Building Data Type Archives This task is often done manually from the Ghidra GUI, and the archives included in our official build require a fair bit of fine tuning. -From a CodeBrowser window, select File -> Parse C Source. +From a CodeBrowser window, select __File -> Parse C Source__. From here you can create and configure parsing profiles, which lists headers and pre-processor options. -Then, click "Parse to File" to create the Data Type Archive. -The result can be added to an installation or source tree by copying it to `Ghidra/Features/Base/data/typeinfo`. +Then, click _Parse to File_ to create the Data Type Archive. +The result can be added to an installation or source tree by copying it to `~/git/ghidra/Ghidra/Features/Base/data/typeinfo`. ### Building FID Databases @@ -340,11 +328,11 @@ You will first need to import the relevant libraries from which you'd like to pr This is often a set of libraries from an SDK. We include a variety of Visual Studio platforms in the official build. -From a CodeBrowser window, select File -> Configure. +From a CodeBrowser window, select __File -> Configure__. Enable the "Function ID" plugins, and close the dialog. -Now, from the CodeBrowser window, select Tools -> Function ID -> Create new empty FidDb. +Now, from the CodeBrowser window, select __Tools -> Function ID -> Create new empty FidDb__. Choose a destination file. -Now, select Tools -> Function ID -> Populate FidDb from programs. +Now, select __Tools -> Function ID -> Populate FidDb__ from programs. Fill out the options appropriately and click OK. -If you'd like some details of our fine tuning, take a look at `Ghidra/Features/FunctionID/building_fid.txt`. +If you'd like some details of our fine tuning, take a look at `~/git/ghidra/Ghidra/Features/FunctionID/data/building_fid.txt`. diff --git a/build.gradle b/build.gradle index a007161b2e..d61f686cea 100644 --- a/build.gradle +++ b/build.gradle @@ -39,6 +39,19 @@ allprojects { } } +/********************************************************************************* + * Use flat directory-style repository if flatRepo directory is present. + *********************************************************************************/ +if (file("flatRepo").isDirectory()) { + allprojects { + repositories { + mavenCentral() + jcenter() + flatDir name: "flat", dirs:["$rootProject.projectDir/flatRepo"] + } + } +} + /********************************************************************************* * load properties from Ghidra/application.properties file *********************************************************************************/ diff --git a/gradle/init.gradle b/gradle/init.gradle index 68ea8d3950..d9aaa69895 100644 --- a/gradle/init.gradle +++ b/gradle/init.gradle @@ -43,7 +43,7 @@ import org.apache.commons.io.filefilter.*; ext.HOME_DIR = System.getProperty('user.home') ext.REPO_DIR = ((Script)this).buildscript.getSourceFile().getParentFile().getParentFile() -ext.FLAT_REPO_DIR = new File(HOME_DIR, "flatRepo") +ext.FLAT_REPO_DIR = new File(REPO_DIR, "flatRepo") ext.DOWNLOADS_DIR = new File(REPO_DIR, "build/downloads") // Stores the size of the file being downloaded (for formatting print statements) @@ -83,7 +83,6 @@ initscript { // This is where the real flow of the script starts... try { createDirs() - createConfigFile() populateFlatRepo() } finally { @@ -102,30 +101,6 @@ def createDirs() { } } -/** - * Creates the repos.gradle configuration file that tells Gradle - * where to look for dependencies. This ensures that Gradle will - * find the jars we store in the local flat repo. - */ -def createConfigFile() { - - ext.repoConfigDir = new File(HOME_DIR + "/.gradle/init.d") - ext.repoConfigFile = new File(repoConfigDir, "repos.gradle") - - if (!repoConfigDir.exists()) { - repoConfigDir.mkdirs() - } - - repoConfigFile.write("ext.HOME = System.getProperty('user.home')") - repoConfigFile.append("\nallprojects {") - repoConfigFile.append("\n\trepositories {") - repoConfigFile.append("\n\t\tmavenCentral()") - repoConfigFile.append("\n\t\tjcenter()") - repoConfigFile.append('\n\t\tflatDir name: "flat", dirs:["$HOME/flatRepo"]') - repoConfigFile.append("\n\t}") - repoConfigFile.append("\n}") -} - /** * Downloads a file from a URL. If there is a problem connecting to the given * URL the attempt will be retried NUM_RETRIES times before failing. From 037b5df87b66a4209bf80f12aa1ee18d9f2e15e7 Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Thu, 27 Jun 2019 11:51:29 -0400 Subject: [PATCH 9/9] GT-2897: Renaming initialization script. --- DevGuide.md | 6 ++--- gradle/certification.manifest | 2 +- .../{init.gradle => fetchDependencies.gradle} | 25 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) rename gradle/{init.gradle => fetchDependencies.gradle} (93%) diff --git a/DevGuide.md b/DevGuide.md index 760cc1843c..6862122917 100644 --- a/DevGuide.md +++ b/DevGuide.md @@ -86,10 +86,10 @@ or manually by downloading the required dependencies. Choose one of the two fol The flat directory-style repository can be setup automatically by running a simple Gradle script. Navigate to `~/git/ghidra` and run the following: ``` -gradle --init-script gradle/init.gradle tasks +gradle --init-script gradle/fetchDependencies.gradle init ``` -The Gradle task to be executed, in this case _tasks_, is unimportant. The point is to have Gradle execute -the `init.gradle` script. If it ran correctly you will have a new `~/git/ghidra/flatRepo/` +The Gradle task to be executed, in this case _init_, is unimportant. The point is to have Gradle execute +the `fetchDependencies.gradle` script. If it ran correctly you will have a new `~/git/ghidra/flatRepo/` directory populated with the following jar files: * AXMLPrinter2 * csframework diff --git a/gradle/certification.manifest b/gradle/certification.manifest index fe96782ed1..f73c58ae81 100644 --- a/gradle/certification.manifest +++ b/gradle/certification.manifest @@ -4,8 +4,8 @@ distributableGPLModule.gradle||GHIDRA||||END| distributableGhidraExtension.gradle||GHIDRA||||END| distributableGhidraModule.gradle||GHIDRA||||END| externalGhidraExtension.gradle||GHIDRA||||END| +fetchDependencies.gradle||GHIDRA||||END| helpProject.gradle||GHIDRA||||END| -init.gradle||GHIDRA||||END| jacocoProject.gradle||GHIDRA||||END| javaProject.gradle||GHIDRA||||END| javaTestProject.gradle||GHIDRA||||END| diff --git a/gradle/init.gradle b/gradle/fetchDependencies.gradle similarity index 93% rename from gradle/init.gradle rename to gradle/fetchDependencies.gradle index d9aaa69895..63c7eb694a 100644 --- a/gradle/init.gradle +++ b/gradle/fetchDependencies.gradle @@ -1,16 +1,16 @@ /******************************************************************************* - * init.gradle * - * * - * Sets up the gradle configuration for external users and downloads * - * any required dependencies that aren't available in the * - * other online repositories (eg: maven). This should be run * + * fetchDependencies.gradle * + * * + * Fetches/downloads required dependencies that aren't available in the * + * standard online repositories (eg: maven) and configures a flat * + * directory-style respository that points to them. This should be run * * immediately after cloning the Ghidra repository before any other gradle * * tasks are run. * * * * Specifically, this task: * * * * 1. Downloads various dependencies required by the ghidra build and * - * puts them in /ghidra/downloads/. From here they are * + * puts them in /build/downloads/. From here they are * * unzipped and/or copied to their final locations. The files to be * * downloaded: * * - dex-tools-2.0.zip * @@ -20,18 +20,17 @@ * - cdt-8.6.0.zip * * - PyDev 6.3.1.zip * * * - * 2. Creates a gradle configuration file (repos.config) in * - * /.gradle/init.d/. This contains repository * - * information used by gradle to find dependencies (it points * - * gradle to the flatRepo location created above). * + * 2. Creates a directory at /flatRepo which is used as a * + * flat directory-style respository for the files extracted above. * * * * usage: from the command line in the main ghidra repository * * directory, run the following: * * * - * gradle --init-script gradle/init.gradle * + * gradle --init-script gradle/fetchDependencies.gradle init * * * - * Note: When running the script, files will only be downloaded if * - * necessary (eg: they are not already in the downloads/ folder). * * + * Note: When running the script, files will only be downloaded if * + * necessary (eg: they are not already in the build/downloads/ * + * directory). * * * *******************************************************************************/