From a9897f277972145a58c36dfde6cffedd8b2247a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Pluta?= Date: Thu, 11 Apr 2019 10:30:34 +0200 Subject: [PATCH] 13: Add debug option --- .../bartek/esa/analyser/apk/ApkAnalyser.java | 6 +-- .../bartek/esa/analyser/core/Analyser.java | 12 +++--- .../esa/analyser/source/SourceAnalyser.java | 4 +- .../bartek/esa/cli/model/CliArgsOptions.java | 1 + .../bartek/esa/cli/parser/CliArgsParser.java | 10 +++++ .../esa/core/executor/PluginExecutor.java | 10 +++-- .../bartek/esa/core/model/object/Issue.java | 14 ++++++- .../esa/decompiler/decompiler/Decompiler.java | 24 ++++++------ .../decompiler/process/ProcessExecutor.java | 37 ++++++++++++++++++- .../dispatcher/MethodDispatcher.java | 6 ++- .../bartek/esa/dispatcher/model/Action.java | 2 +- .../formatter/formatter/ColorFormatter.java | 1 + .../formatter/formatter/SimpleFormatter.java | 1 + 13 files changed, 95 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/bartek/esa/analyser/apk/ApkAnalyser.java b/src/main/java/com/bartek/esa/analyser/apk/ApkAnalyser.java index 67dca4a..ab4b5fc 100644 --- a/src/main/java/com/bartek/esa/analyser/apk/ApkAnalyser.java +++ b/src/main/java/com/bartek/esa/analyser/apk/ApkAnalyser.java @@ -29,10 +29,10 @@ public class ApkAnalyser extends Analyser { } @Override - protected String prepareSources(String source) { + protected String prepareSources(String source, boolean debug) { checkIfSourceIsApkFile(source); System.out.println("Decompiling APK..."); - return decompiler.decompile(new File(source)).getAbsolutePath(); + return decompiler.decompile(new File(source), debug).getAbsolutePath(); } private void checkIfSourceIsApkFile(String source) { @@ -57,7 +57,7 @@ public class ApkAnalyser extends Analyser { } @Override - protected void performCleaning(String source) { + protected void performCleaning(String source, boolean debug) { fileCleaner.deleteRecursively(new File(source)); } } diff --git a/src/main/java/com/bartek/esa/analyser/core/Analyser.java b/src/main/java/com/bartek/esa/analyser/core/Analyser.java index a4ba6b6..1e9bd0f 100644 --- a/src/main/java/com/bartek/esa/analyser/core/Analyser.java +++ b/src/main/java/com/bartek/esa/analyser/core/Analyser.java @@ -24,18 +24,18 @@ public abstract class Analyser { this.fileProvider = fileProvider; } - public Set analyse(String source, Set pluginCodes, Set excludeCodes) { - String newSource = prepareSources(source); + public Set analyse(String source, Set pluginCodes, Set excludeCodes, boolean debug) { + String newSource = prepareSources(source, debug); File manifest = getManifest(newSource); Set files = getFiles(newSource); Set selectedPlugins = getPlugins(pluginCodes, excludeCodes); - Set issues = pluginExecutor.executeForFiles(manifest, files, selectedPlugins); - performCleaning(newSource); + Set issues = pluginExecutor.executeForFiles(manifest, files, selectedPlugins, debug); + performCleaning(newSource, debug); return issues; } - protected abstract String prepareSources(String source); + protected abstract String prepareSources(String source, boolean debug); protected abstract String getAndroidManifestGlob(); @@ -43,7 +43,7 @@ public abstract class Analyser { protected abstract String getLayoutGlob(); - protected abstract void performCleaning(String source); + protected abstract void performCleaning(String source, boolean debug); private File getManifest(String source) { diff --git a/src/main/java/com/bartek/esa/analyser/source/SourceAnalyser.java b/src/main/java/com/bartek/esa/analyser/source/SourceAnalyser.java index 97a9fed..f55650a 100644 --- a/src/main/java/com/bartek/esa/analyser/source/SourceAnalyser.java +++ b/src/main/java/com/bartek/esa/analyser/source/SourceAnalyser.java @@ -19,7 +19,7 @@ public class SourceAnalyser extends Analyser { } @Override - protected String prepareSources(String source) { + protected String prepareSources(String source, boolean debug) { checkIfSourceIsDirectory(source); return source; } @@ -46,7 +46,7 @@ public class SourceAnalyser extends Analyser { } @Override - protected void performCleaning(String source) { + protected void performCleaning(String source, boolean debug) { // do nothing } } diff --git a/src/main/java/com/bartek/esa/cli/model/CliArgsOptions.java b/src/main/java/com/bartek/esa/cli/model/CliArgsOptions.java index 074b5fb..a9ff835 100644 --- a/src/main/java/com/bartek/esa/cli/model/CliArgsOptions.java +++ b/src/main/java/com/bartek/esa/cli/model/CliArgsOptions.java @@ -16,6 +16,7 @@ public class CliArgsOptions { private Set plugins; private boolean color; private Set severities; + private boolean debug; public boolean isSourceAnalysis() { return sourceAnalysisDirectory != null; diff --git a/src/main/java/com/bartek/esa/cli/parser/CliArgsParser.java b/src/main/java/com/bartek/esa/cli/parser/CliArgsParser.java index 1dd5d06..b780048 100644 --- a/src/main/java/com/bartek/esa/cli/parser/CliArgsParser.java +++ b/src/main/java/com/bartek/esa/cli/parser/CliArgsParser.java @@ -20,6 +20,7 @@ public class CliArgsParser { private static final String PLUGINS_OPT = "plugins"; private static final String COLOR_OPT = "color"; private static final String SEVERITIES_OPT = "severities"; + private static final String DEBUG_OPT = "debug"; @Inject public CliArgsParser() {} @@ -55,6 +56,7 @@ public class CliArgsParser { .excludes(command.hasOption(EXCLUDE_OPT) ? new HashSet<>(asList(command.getOptionValues(EXCLUDE_OPT))) : emptySet()) .color(command.hasOption(COLOR_OPT)) .severities(command.hasOption(SEVERITIES_OPT) ? new HashSet<>(asList((command.getOptionValues(SEVERITIES_OPT)))) : stream(Severity.values()).map(Severity::name).collect(Collectors.toSet())) + .debug(command.hasOption(DEBUG_OPT)) .build(); } @@ -72,6 +74,7 @@ public class CliArgsParser { options.addOption(help()); options.addOption(color()); options.addOption(severities()); + options.addOption(debug()); return options; } @@ -134,4 +137,11 @@ public class CliArgsParser { .desc("filter output to selected severities(available: " + severities + ")") .build(); } + + private Option debug() { + return Option.builder() + .longOpt(DEBUG_OPT) + .desc("enable debug mode") + .build(); + } } diff --git a/src/main/java/com/bartek/esa/core/executor/PluginExecutor.java b/src/main/java/com/bartek/esa/core/executor/PluginExecutor.java index 3560e12..4c86bf7 100644 --- a/src/main/java/com/bartek/esa/core/executor/PluginExecutor.java +++ b/src/main/java/com/bartek/esa/core/executor/PluginExecutor.java @@ -19,16 +19,18 @@ public class PluginExecutor { this.xmlHelper = xmlHelper; } - public Set executeForFiles(File manifest, Set files, Set plugins) { + public Set executeForFiles(File manifest, Set files, Set plugins, boolean debug) { return files.stream() - .map(file -> executeForFile(manifest, file, plugins)) + .peek(file -> { if(debug) System.out.printf("File: %s", file.getAbsolutePath()); }) + .map(file -> executeForFile(manifest, file, plugins, debug)) .flatMap(Set::stream) .collect(toSet()); } - private Set executeForFile(File manifest, File file, Set plugins) { + private Set executeForFile(File manifest, File file, Set plugins, boolean debug) { Document xmlManifest = xmlHelper.parseXml(manifest); - return plugins.stream() + return plugins.parallelStream() + .peek(plugin -> { if(debug) System.out.printf(" Plugin: %s", plugin.getClass().getCanonicalName()); }) .peek(plugin -> plugin.update(file, xmlManifest)) .filter(plugin -> plugin.supports(file)) .map(Plugin::runForIssues) diff --git a/src/main/java/com/bartek/esa/core/model/object/Issue.java b/src/main/java/com/bartek/esa/core/model/object/Issue.java index 249250f..11ddcd6 100644 --- a/src/main/java/com/bartek/esa/core/model/object/Issue.java +++ b/src/main/java/com/bartek/esa/core/model/object/Issue.java @@ -8,11 +8,23 @@ import java.io.File; @Data @Builder -public class Issue { +public class Issue implements Comparable { private final Class issuer; private final Severity severity; private final String descriptionCode; private final File file; private final Integer lineNumber; private final String line; + + @Override + public int compareTo(Object o) { + Issue another = (Issue) o; + int compByFile = file.compareTo(another.file); + + if(compByFile != 0) { + return compByFile; + } + + return lineNumber - another.lineNumber; + } } diff --git a/src/main/java/com/bartek/esa/decompiler/decompiler/Decompiler.java b/src/main/java/com/bartek/esa/decompiler/decompiler/Decompiler.java index de8d41b..21ca070 100644 --- a/src/main/java/com/bartek/esa/decompiler/decompiler/Decompiler.java +++ b/src/main/java/com/bartek/esa/decompiler/decompiler/Decompiler.java @@ -30,51 +30,51 @@ public class Decompiler { this.fileCleaner = fileCleaner; } - public File decompile(File inputApk) { + public File decompile(File inputApk, boolean debug) { File tmp = fileProvider.createTemporaryDirectory(); File javaDirectory = new File(tmp, JAVA_FILES_DIR); File xmlDirectory = new File(tmp, XML_FILES_DIR); - decompileJavaFiles(inputApk, tmp, javaDirectory); - decompileXmlFiles(inputApk, xmlDirectory); + decompileJavaFiles(inputApk, tmp, javaDirectory, debug); + decompileXmlFiles(inputApk, xmlDirectory, debug); return tmp; } - private void decompileJavaFiles(File inputApk, File tmp, File javaDirectory) { + private void decompileJavaFiles(File inputApk, File tmp, File javaDirectory, boolean debug) { File unzippedApkDirectory = new File(tmp, APK_UNZIPPED_DIR); File jarDirectory = new File(tmp, JAR_FILES_DIR); zipTool.unzipArchive(inputApk, unzippedApkDirectory); Set dexFiles = fileProvider.getGlobMatchedFiles(unzippedApkDirectory.getAbsolutePath(), "**/*.dex"); - convertDexToJar(dexFiles, jarDirectory); + convertDexToJar(dexFiles, jarDirectory, debug); Set jarFiles = fileProvider.getGlobMatchedFiles(jarDirectory.getAbsolutePath(), "**/*.jar"); - decompileJar(jarFiles, javaDirectory); + decompileJar(jarFiles, javaDirectory, debug); fileCleaner.deleteRecursively(unzippedApkDirectory); fileCleaner.deleteRecursively(jarDirectory); } - private void decompileXmlFiles(File inputApk, File target) { + private void decompileXmlFiles(File inputApk, File target, boolean debug) { String[] command = {"apktool", "d", inputApk.getAbsolutePath(), "-o", target.getAbsolutePath()}; - processExecutor.execute(command); + processExecutor.execute(command, debug); } - private void convertDexToJar(Set dexFiles, File target) { + private void convertDexToJar(Set dexFiles, File target, boolean debug) { dexFiles.forEach(dex -> { String jarFilename = FilenameUtils.removeExtension(dex.getName()) + ".jar"; File jarFile = new File(target, jarFilename); String[] command = {"dex2jar", dex.getAbsolutePath(), "-o", jarFile.getAbsolutePath()}; - processExecutor.execute(command); + processExecutor.execute(command, debug); }); } - private void decompileJar(Set jarFiles, File target) { + private void decompileJar(Set jarFiles, File target, boolean debug) { jarFiles.forEach(jar -> { String[] command = {"cfr", jar.getAbsolutePath(), "--outputdir", target.getAbsolutePath()}; - processExecutor.execute(command); + processExecutor.execute(command, debug); }); } } diff --git a/src/main/java/com/bartek/esa/decompiler/process/ProcessExecutor.java b/src/main/java/com/bartek/esa/decompiler/process/ProcessExecutor.java index 1b59dbd..e4e2b5d 100644 --- a/src/main/java/com/bartek/esa/decompiler/process/ProcessExecutor.java +++ b/src/main/java/com/bartek/esa/decompiler/process/ProcessExecutor.java @@ -4,6 +4,9 @@ import com.bartek.esa.error.EsaException; import io.vavr.control.Try; import javax.inject.Inject; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; public class ProcessExecutor { @@ -12,19 +15,49 @@ public class ProcessExecutor { } - public void execute(String[] command) { + public void execute(String[] command, boolean debug) { + printCommandLine(command, debug); Process process = Try.of(() -> Runtime.getRuntime().exec(command)) .getOrElseThrow(EsaException::new); + printStdOutAndStdErrFromProcess(debug, process); waitForProcess(process); checkExitValue(process, command[0]); } + private void printCommandLine(String[] command, boolean debug) { + if(debug) { + System.out.println(String.join(" ", command)); + } + } + + private void printStdOutAndStdErrFromProcess(boolean debug, Process process) { + if(debug) { + BufferedReader stdout = new BufferedReader(new InputStreamReader(process.getInputStream())); + BufferedReader stderr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String line; + try { + while ((line = stdout.readLine()) != null) { + System.out.println(line); + } + + while ((line = stderr.readLine()) != null) { + System.err.println(line); + } + + stdout.close(); + stderr.close(); + } catch (IOException e) { + throw new EsaException(e); + } + } + } + private void waitForProcess(Process process) { Try.run(process::waitFor).getOrElseThrow(EsaException::new); } private void checkExitValue(Process process, String commandName) { - if(process.exitValue() != 0) { + if (process.exitValue() != 0) { throw new EsaException("'" + commandName + "' process has finished with non-zero code. Interrupting..."); } } diff --git a/src/main/java/com/bartek/esa/dispatcher/dispatcher/MethodDispatcher.java b/src/main/java/com/bartek/esa/dispatcher/dispatcher/MethodDispatcher.java index 5b00209..8c294cd 100644 --- a/src/main/java/com/bartek/esa/dispatcher/dispatcher/MethodDispatcher.java +++ b/src/main/java/com/bartek/esa/dispatcher/dispatcher/MethodDispatcher.java @@ -20,7 +20,8 @@ public class MethodDispatcher { return actions.getSourceAnalysis().perform( options.getSourceAnalysisDirectory(), options.getPlugins(), - options.getExcludes() + options.getExcludes(), + options.isDebug() ); } @@ -28,7 +29,8 @@ public class MethodDispatcher { return actions.getApkAudit().perform( options.getApkAuditFile(), options.getPlugins(), - options.getExcludes() + options.getExcludes(), + options.isDebug() ); } diff --git a/src/main/java/com/bartek/esa/dispatcher/model/Action.java b/src/main/java/com/bartek/esa/dispatcher/model/Action.java index cf5af89..32795e7 100644 --- a/src/main/java/com/bartek/esa/dispatcher/model/Action.java +++ b/src/main/java/com/bartek/esa/dispatcher/model/Action.java @@ -7,5 +7,5 @@ import java.util.Set; @FunctionalInterface public interface Action { - Set perform(String source, Set plugins, Set excludes); + Set perform(String source, Set plugins, Set excludes, boolean debug); } diff --git a/src/main/java/com/bartek/esa/formatter/formatter/ColorFormatter.java b/src/main/java/com/bartek/esa/formatter/formatter/ColorFormatter.java index 34ee131..d7ac99d 100644 --- a/src/main/java/com/bartek/esa/formatter/formatter/ColorFormatter.java +++ b/src/main/java/com/bartek/esa/formatter/formatter/ColorFormatter.java @@ -34,6 +34,7 @@ public class ColorFormatter implements Formatter { } String format = issues.stream() + .sorted() .map(this::format) .collect(Collectors.joining()); diff --git a/src/main/java/com/bartek/esa/formatter/formatter/SimpleFormatter.java b/src/main/java/com/bartek/esa/formatter/formatter/SimpleFormatter.java index aae9181..8294849 100644 --- a/src/main/java/com/bartek/esa/formatter/formatter/SimpleFormatter.java +++ b/src/main/java/com/bartek/esa/formatter/formatter/SimpleFormatter.java @@ -25,6 +25,7 @@ public class SimpleFormatter implements Formatter { } String format = issues.stream() + .sorted() .map(this::format) .collect(Collectors.joining());