jessikitty/jadx
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(smali-input): compile one smali file at a time to avoid 64k limit (#2158)

Browse Source
This commit is contained in:
Skylot
2024-04-23 22:14:59 +01:00
parent ce527ed753
commit b80f32a36f
8 changed files with 204 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jadx-core/src/main/java/jadx/api/plugins/options/impl/BasePluginOptionsBuilder.java
+8
View File
@@ -64,6 +64,14 @@ public abstract class BasePluginOptionsBuilder implements JadxPluginOptions {
.parser(v -> v));
}
public OptionBuilder<Integer> intOption(String name) {
return addOption(
new OptionData<Integer>(name)
.type(OptionType.NUMBER)
.formatter(Object::toString)
.parser(Integer::parseInt));
}
public <E extends Enum<?>> OptionBuilder<E> enumOption(String name, E[] values, Function<String, E> valueOf) {
return addOption(
new OptionData<E>(name)
jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/DexInputPlugin.java
+9
View File
@@ -5,6 +5,7 @@ import java.io.InputStream;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.jetbrains.annotations.Nullable;
@@ -14,6 +15,7 @@ import jadx.api.plugins.JadxPluginInfo;
import jadx.api.plugins.input.ICodeLoader;
import jadx.api.plugins.input.data.impl.EmptyCodeLoader;
import jadx.api.plugins.utils.CommonFileUtils;
import jadx.plugins.input.dex.utils.IDexData;
public class DexInputPlugin implements JadxPlugin {
public static final String PLUGIN_ID = "dex-input";
@@ -57,4 +59,11 @@ public class DexInputPlugin implements JadxPlugin {
throw new DexException("Failed to read input stream", e);
}
}
public ICodeLoader loadDexData(List<IDexData> list) {
List<DexReader> readers = list.stream()
.map(data -> loader.loadDexReader(data.getFileName(), data.getContent()))
.collect(Collectors.toList());
return new DexLoadResult(readers, null);
}
}
jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/utils/IDexData.java
+8
View File
@@ -0,0 +1,8 @@
package jadx.plugins.input.dex.utils;
public interface IDexData {
String getFileName();
byte[] getContent();
}
jadx-plugins/jadx-dex-input/src/main/java/jadx/plugins/input/dex/utils/SimpleDexData.java
+28
View File
@@ -0,0 +1,28 @@
package jadx.plugins.input.dex.utils;
import java.util.Objects;
public class SimpleDexData implements IDexData {
private final String fileName;
private final byte[] content;
public SimpleDexData(String fileName, byte[] content) {
this.fileName = Objects.requireNonNull(fileName);
this.content = Objects.requireNonNull(content);
}
@Override
public String getFileName() {
return fileName;
}
@Override
public byte[] getContent() {
return content;
}
@Override
public String toString() {
return "DexData{" + fileName + ", size=" + content.length + '}';
}
}
jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliConvert.java
+58 -51
View File
@@ -1,79 +1,100 @@
package jadx.plugins.input.smali;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Reader;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.android.tools.smali.smali.Smali;
import com.android.tools.smali.smali.SmaliOptions;
public class SmaliConvert implements Closeable {
import jadx.plugins.input.dex.utils.IDexData;
import jadx.plugins.input.dex.utils.SimpleDexData;
public class SmaliConvert {
private static final Logger LOG = LoggerFactory.getLogger(SmaliConvert.class);
@Nullable
private Path tmpDex;
private final List<IDexData> dexData = new ArrayList<>();
public boolean execute(List<Path> input) {
public boolean execute(List<Path> input, SmaliInputOptions options) {
List<Path> smaliFiles = filterSmaliFiles(input);
if (smaliFiles.isEmpty()) {
return false;
}
LOG.debug("Compiling smali files: {}", smaliFiles.size());
try {
this.tmpDex = Files.createTempFile("jadx-", ".dex");
if (compileSmali(tmpDex, smaliFiles)) {
return true;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
collectSystemErrors(out, () -> compile(smaliFiles, options));
boolean success = out.size() == 0;
if (!success) {
LOG.error("Smali error:\n{}", out);
}
} catch (Exception e) {
LOG.error("Smali process error", e);
}
close();
return false;
return !dexData.isEmpty();
}
private static boolean compileSmali(Path output, List<Path> inputFiles) throws IOException {
SmaliOptions options = new SmaliOptions();
options.outputDexFile = output.toAbsolutePath().toString();
options.verboseErrors = true;
options.apiLevel = 27; // TODO: add as plugin option
@SuppressWarnings("ResultOfMethodCallIgnored")
private void compile(List<Path> inputFiles, SmaliInputOptions options) {
SmaliOptions smaliOptions = new SmaliOptions();
smaliOptions.apiLevel = options.getApiLevel();
smaliOptions.verboseErrors = true;
smaliOptions.allowOdexOpcodes = false;
smaliOptions.printTokens = false;
List<String> inputFileNames = inputFiles.stream()
.map(p -> p.toAbsolutePath().toString())
.distinct()
.collect(Collectors.toList());
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
boolean result = collectSystemErrors(out, () -> Smali.assemble(options, inputFileNames));
if (!result) {
LOG.error("Smali compilation error:\n{}", out);
int threads = options.getThreads();
LOG.debug("Compiling smali files: {}, threads: {}", inputFiles.size(), threads);
long start = System.currentTimeMillis();
if (threads == 1) {
for (Path inputFile : inputFiles) {
assemble(inputFile, smaliOptions);
}
return result;
} else {
try {
ExecutorService executor = Executors.newFixedThreadPool(threads);
for (Path inputFile : inputFiles) {
executor.execute(() -> assemble(inputFile, smaliOptions));
}
executor.shutdown();
executor.awaitTermination(1, TimeUnit.DAYS);
} catch (InterruptedException e) {
LOG.error("Smali compile interrupted", e);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Smali compile done in: {}ms", System.currentTimeMillis() - start);
}
}
private static boolean collectSystemErrors(OutputStream out, Callable<Boolean> exec) {
private void assemble(Path inputFile, SmaliOptions smaliOptions) {
String fileName = inputFile.toAbsolutePath().toString();
try (Reader reader = Files.newBufferedReader(inputFile)) {
byte[] assemble = SmaliUtils.assemble(reader, smaliOptions);
dexData.add(new SimpleDexData(fileName, assemble));
} catch (Exception e) {
throw new RuntimeException("Fail to compile: " + fileName, e);
}
}
private static void collectSystemErrors(OutputStream out, Runnable exec) {
PrintStream systemErr = System.err;
try (PrintStream err = new PrintStream(out)) {
System.setErr(err);
try {
return exec.call();
exec.run();
} catch (Exception e) {
e.printStackTrace(err);
return false;
}
} finally {
System.setErr(systemErr);
@@ -87,21 +108,7 @@ public class SmaliConvert implements Closeable {
.collect(Collectors.toList());
}
public List<Path> getDexFiles() {
if (tmpDex == null) {
return Collections.emptyList();
}
return Collections.singletonList(tmpDex);
}
@Override
public void close() {
try {
if (tmpDex != null) {
Files.deleteIfExists(tmpDex);
}
} catch (Exception e) {
LOG.error("Failed to remove tmp dex file: {}", tmpDex, e);
}
public List<IDexData> getDexData() {
return dexData;
}
}
jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliInputOptions.java
+29
View File
@@ -0,0 +1,29 @@
package jadx.plugins.input.smali;
import jadx.api.plugins.options.impl.BasePluginOptionsBuilder;
public class SmaliInputOptions extends BasePluginOptionsBuilder {
private int apiLevel;
private int threads; // use jadx global threads count option
@Override
public void registerOptions() {
intOption(SmaliInputPlugin.PLUGIN_ID + ".api-level")
.description("Android API level")
.defaultValue(27)
.setter(v -> apiLevel = v);
}
public int getApiLevel() {
return apiLevel;
}
public int getThreads() {
return threads;
}
public void setThreads(int threads) {
this.threads = threads;
}
}
jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliInputPlugin.java
+10 -5
View File
@@ -3,26 +3,31 @@ package jadx.plugins.input.smali;
import jadx.api.plugins.JadxPlugin;
import jadx.api.plugins.JadxPluginContext;
import jadx.api.plugins.JadxPluginInfo;
import jadx.api.plugins.data.JadxPluginRuntimeData;
import jadx.api.plugins.input.data.impl.EmptyCodeLoader;
import jadx.plugins.input.dex.DexInputPlugin;
public class SmaliInputPlugin implements JadxPlugin {
public static final String PLUGIN_ID = "smali-input";
private final SmaliInputOptions options = new SmaliInputOptions();
@Override
public JadxPluginInfo getPluginInfo() {
return new JadxPluginInfo("smali-input", "Smali Input", "Load .smali files");
return new JadxPluginInfo(PLUGIN_ID, "Smali Input", "Load .smali files");
}
@Override
public void init(JadxPluginContext context) {
JadxPluginRuntimeData dexInput = context.plugins().getById(DexInputPlugin.PLUGIN_ID);
context.registerOptions(options);
options.setThreads(context.getArgs().getThreadsCount());
DexInputPlugin dexInput = context.plugins().getInstance(DexInputPlugin.class);
context.addCodeInput(input -> {
SmaliConvert convert = new SmaliConvert();
if (!convert.execute(input)) {
if (!convert.execute(input, options)) {
return EmptyCodeLoader.INSTANCE;
}
return dexInput.loadCodeFiles(convert.getDexFiles(), convert);
return dexInput.loadDexData(convert.getDexData());
});
}
}
jadx-plugins/jadx-smali-input/src/main/java/jadx/plugins/input/smali/SmaliUtils.java
+54
View File
@@ -0,0 +1,54 @@
package jadx.plugins.input.smali;
import java.io.IOException;
import java.io.Reader;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenSource;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import com.android.tools.smali.dexlib2.Opcodes;
import com.android.tools.smali.dexlib2.writer.builder.DexBuilder;
import com.android.tools.smali.dexlib2.writer.io.MemoryDataStore;
import com.android.tools.smali.smali.LexerErrorInterface;
import com.android.tools.smali.smali.SmaliOptions;
import com.android.tools.smali.smali.smaliFlexLexer;
import com.android.tools.smali.smali.smaliParser;
import com.android.tools.smali.smali.smaliTreeWalker;
/**
* Utility methods to assemble smali to in-memory buffer.
* This implementation uses smali library internal classes.
*/
public class SmaliUtils {
@SuppressWarnings("ExtractMethodRecommender")
public static byte[] assemble(Reader reader, SmaliOptions options) throws IOException, RecognitionException {
LexerErrorInterface lexer = new smaliFlexLexer(reader, options.apiLevel);
CommonTokenStream tokens = new CommonTokenStream((TokenSource) lexer);
smaliParser parser = new smaliParser(tokens);
parser.setVerboseErrors(options.verboseErrors);
parser.setAllowOdex(options.allowOdexOpcodes);
parser.setApiLevel(options.apiLevel);
smaliParser.smali_file_return parseResult = parser.smali_file();
if (parser.getNumberOfSyntaxErrors() > 0 || lexer.getNumberOfSyntaxErrors() > 0) {
throw new RuntimeException("Parse error");
}
CommonTreeNodeStream treeStream = new CommonTreeNodeStream(parseResult.getTree());
treeStream.setTokenStream(tokens);
DexBuilder dexBuilder = new DexBuilder(Opcodes.forApi(options.apiLevel));
smaliTreeWalker dexGen = new smaliTreeWalker(treeStream);
dexGen.setApiLevel(options.apiLevel);
dexGen.setVerboseErrors(options.verboseErrors);
dexGen.setDexBuilder(dexBuilder);
dexGen.smali_file();
if (dexGen.getNumberOfSyntaxErrors() > 0) {
throw new RuntimeException("Compile error");
}
MemoryDataStore dataStore = new MemoryDataStore();
dexBuilder.writeTo(dataStore);
return dataStore.getData();
}
}