fix: update class set to Android 34, add apache http client support for api level 28+ (PR #1927)

* fix(cli): fix jcst converter

* extend jcst format, update class set to Android 34, add optional android libs

* fix(gradle): add apache http client support for api level 28+

* don't require existing core.jcst file for convert cls set, improve performance

---------

Co-authored-by: Skylot <skylot@gmail.com>

jadx-cli/src/main/java/jadx/cli/clst/ConvertToClsSet.java

@@ -2,7 +2,6 @@ package jadx.cli.clst;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -13,14 +12,9 @@ import org.slf4j.LoggerFactory;
 
 import jadx.api.JadxArgs;
 import jadx.api.JadxDecompiler;
-import jadx.api.plugins.input.ICodeLoader;
-import jadx.api.plugins.input.JadxCodeInput;
-import jadx.api.plugins.loader.JadxBasePluginLoader;
 import jadx.core.clsp.ClsSet;
-import jadx.core.dex.nodes.ClassNode;
 import jadx.core.dex.nodes.RootNode;
-import jadx.core.dex.visitors.SignatureProcessor;
-import jadx.core.plugins.JadxPluginManager;
+import jadx.core.utils.files.FileUtils;
 
 /**
  * Utility class for convert dex or jar to jadx classes set (.jcst)
@@ -30,9 +24,14 @@ public class ConvertToClsSet {
 
 	public static void usage() {
 		LOG.info("<output .jcst or .jar file> <several input dex or jar files> ");
+		LOG.info("Arguments to update core.jcst: "
+				+ "<jadx root>/jadx-core/src/main/resources/clst/core.jcst "
+				+ "<sdk_root>/platforms/android-<api level>/android.jar"
+				+ "<sdk_root>/platforms/android-<api level>/optional/android.car.jar "
+				+ "<sdk_root>/platforms/android-<api level>/optional/org.apache.http.legacy.jar");
 	}
 
-	public static void main(String[] args) throws Exception {
+	public static void main(String[] args) {
 		if (args.length < 2) {
 			usage();
 			System.exit(1);
@@ -41,25 +40,22 @@ public class ConvertToClsSet {
 		Path output = inputPaths.remove(0);
 
 		JadxArgs jadxArgs = new JadxArgs();
+		jadxArgs.setInputFiles(FileUtils.toFiles(inputPaths));
+
+		// disable not needed passes executed at prepare stage
+		jadxArgs.setDeobfuscationOn(false);
 		jadxArgs.setRenameFlags(EnumSet.noneOf(JadxArgs.RenameEnum.class));
+		jadxArgs.setUseSourceNameAsClassAlias(false);
+		jadxArgs.setMoveInnerClasses(false);
+		jadxArgs.setInlineAnonymousClasses(false);
+		jadxArgs.setInlineMethods(false);
+
+		// don't require/load class set file
+		jadxArgs.setLoadJadxClsSetFile(false);
+
 		try (JadxDecompiler decompiler = new JadxDecompiler(jadxArgs)) {
-			JadxPluginManager pluginManager = decompiler.getPluginManager();
-			pluginManager.load(new JadxBasePluginLoader());
-			pluginManager.initResolved();
-			List<ICodeLoader> loadedInputs = new ArrayList<>();
-			for (JadxCodeInput inputPlugin : pluginManager.getCodeInputs()) {
-				loadedInputs.add(inputPlugin.loadFiles(inputPaths));
-			}
+			decompiler.load();
 			RootNode root = decompiler.getRoot();
-			root.loadClasses(loadedInputs);
-
-			// from pre-decompilation stage run only SignatureProcessor
-			SignatureProcessor signatureProcessor = new SignatureProcessor();
-			signatureProcessor.init(root);
-			for (ClassNode classNode : root.getClasses()) {
-				signatureProcessor.visit(classNode);
-			}
-
 			ClsSet set = new ClsSet(root);
 			set.loadFrom(root);
 			set.save(output);

jadx-core/src/main/java/jadx/api/JadxArgs.java

@@ -157,6 +157,8 @@ public class JadxArgs implements Closeable {
 
 	private JadxPluginLoader pluginLoader = new JadxBasePluginLoader();
 
+	private boolean loadJadxClsSetFile = true;
+
 	public JadxArgs() {
 		// use default options
 	}
@@ -652,6 +654,14 @@ public class JadxArgs implements Closeable {
 		this.pluginLoader = pluginLoader;
 	}
 
+	public boolean isLoadJadxClsSetFile() {
+		return loadJadxClsSetFile;
+	}
+
+	public void setLoadJadxClsSetFile(boolean loadJadxClsSetFile) {
+		this.loadJadxClsSetFile = loadJadxClsSetFile;
+	}
+
 	/**
 	 * Hash of all options that can change result code
 	 */

jadx-core/src/main/java/jadx/core/clsp/ClsSet.java

@@ -47,10 +47,11 @@ public class ClsSet {
 
 	private static final String CLST_EXTENSION = ".jcst";
 	private static final String CLST_FILENAME = "core" + CLST_EXTENSION;
+
 	private static final String CLST_PATH = "/clst/" + CLST_FILENAME;
 
 	private static final String JADX_CLS_SET_HEADER = "jadx-cst";
-	private static final int VERSION = 3;
+	private static final int VERSION = 4;
 
 	private static final String STRING_CHARSET = "US-ASCII";
 
@@ -97,7 +98,9 @@ public class ClsSet {
 			ArgType clsType = cls.getClassInfo().getType();
 			String clsRawName = clsType.getObject();
 			cls.load();
-			ClspClass nClass = new ClspClass(clsType, k);
+
+			ClspClassSource source = getClspClassSource(cls);
+			ClspClass nClass = new ClspClass(clsType, k, source);
 			if (names.put(clsRawName, nClass) != null) {
 				throw new JadxRuntimeException("Duplicate class: " + clsRawName);
 			}
@@ -118,6 +121,17 @@ public class ClsSet {
 		}
 	}
 
+	private static ClspClassSource getClspClassSource(ClassNode cls) {
+		String inputFileName = cls.getClsData().getInputFileName();
+		int idx = inputFileName.indexOf(':');
+		String sourceFile = inputFileName.substring(0, idx);
+		ClspClassSource source = ClspClassSource.getClspClassSource(sourceFile);
+		if (source == ClspClassSource.APP) {
+			throw new JadxRuntimeException("Unexpected input file: " + inputFileName);
+		}
+		return source;
+	}
+
 	private List<ClspMethod> getMethodsDetails(ClassNode cls) {
 		List<MethodNode> methodsList = cls.getMethods();
 		List<ClspMethod> methods = new ArrayList<>(methodsList.size());
@@ -217,6 +231,7 @@ public class ClsSet {
 		Map<String, ClspClass> names = new HashMap<>(classes.length);
 		out.writeInt(classes.length);
 		for (ClspClass cls : classes) {
+			writeUnsignedByte(out, cls.getSource().ordinal());
 			String clsName = cls.getName();
 			writeString(out, clsName);
 			names.put(clsName, cls);
@@ -342,9 +357,14 @@ public class ClsSet {
 			}
 			int clsCount = in.readInt();
 			classes = new ClspClass[clsCount];
+			ClspClassSource[] clspClassSources = ClspClassSource.values();
 			for (int i = 0; i < clsCount; i++) {
+				int source = readUnsignedByte(in);
+				if (source < 0 || source > clspClassSources.length) {
+					throw new DecodeException("Wrong jadx source identifier");
+				}
 				String name = readString(in);
-				classes[i] = new ClspClass(ArgType.object(name), i);
+				classes[i] = new ClspClass(ArgType.object(name), i, clspClassSources[source]);
 			}
 			for (int i = 0; i < clsCount; i++) {
 				ClspClass nClass = classes[i];

jadx-core/src/main/java/jadx/core/clsp/ClspClass.java

@@ -20,9 +20,18 @@ public class ClspClass {
 	private Map<String, ClspMethod> methodsMap = Collections.emptyMap();
 	private List<ArgType> typeParameters = Collections.emptyList();
 
+	private ClspClassSource source;
+
 	public ClspClass(ArgType clsType, int id) {
 		this.clsType = clsType;
 		this.id = id;
+		this.source = ClspClassSource.APP;
+	}
+
+	public ClspClass(ArgType clsType, int id, ClspClassSource source) {
+		this.clsType = clsType;
+		this.id = id;
+		this.source = source;
 	}
 
 	public String getName() {
@@ -76,6 +85,10 @@ public class ClspClass {
 		this.typeParameters = typeParameters;
 	}
 
+	public ClspClassSource getSource() {
+		return this.source;
+	}
+
 	@Override
 	public int hashCode() {
 		return clsType.hashCode();

jadx-core/src/main/java/jadx/core/clsp/ClspClassSource.java

@@ -0,0 +1,27 @@
+package jadx.core.clsp;
+
+public enum ClspClassSource {
+	APP(""),
+	CORE("android.jar"),
+	ANDROID_CAR("android.car.jar"),
+	APACHE_HTTP_LEGACY_CLIENT("org.apache.http.legacy.jar");
+
+	private final String jarFile;
+
+	ClspClassSource(String jarFile) {
+		this.jarFile = jarFile;
+	}
+
+	public String getJarFile() {
+		return jarFile;
+	}
+
+	public static ClspClassSource getClspClassSource(String jarFile) {
+		for (ClspClassSource classSource : ClspClassSource.values()) {
+			if (classSource.getJarFile().equals(jarFile)) {
+				return classSource;
+			}
+		}
+		return APP;
+	}
+}

jadx-core/src/main/java/jadx/core/clsp/ClspGraph.java

@@ -38,7 +38,7 @@ public class ClspGraph {
 		this.root = rootNode;
 	}
 
-	public void load() throws IOException, DecodeException {
+	public void loadClsSetFile() throws IOException, DecodeException {
 		ClsSet set = new ClsSet(root);
 		set.loadFromClstFile();
 		addClasspath(set);
@@ -55,7 +55,7 @@ public class ClspGraph {
 
 	public void addApp(List<ClassNode> classes) {
 		if (nameMap == null) {
-			throw new JadxRuntimeException("Classpath must be loaded first");
+			nameMap = new HashMap<>(classes.size());
 		}
 		for (ClassNode cls : classes) {
 			addClass(cls);

jadx-core/src/main/java/jadx/core/dex/nodes/RootNode.java

@@ -245,7 +245,9 @@ public class RootNode {
 		try {
 			if (this.clsp == null) {
 				ClspGraph newClsp = new ClspGraph(this);
-				newClsp.load();
+				if (args.isLoadJadxClsSetFile()) {
+					newClsp.loadClsSetFile();
+				}
 				newClsp.addApp(classes);
 				newClsp.initCache();
 				this.clsp = newClsp;

jadx-core/src/main/java/jadx/core/dex/visitors/usage/UsageInfo.java

@@ -8,6 +8,8 @@ import java.util.function.Consumer;
 
 import jadx.api.usage.IUsageInfoData;
 import jadx.api.usage.IUsageInfoVisitor;
+import jadx.core.clsp.ClspClass;
+import jadx.core.clsp.ClspClassSource;
 import jadx.core.dex.instructions.args.ArgType;
 import jadx.core.dex.nodes.ClassNode;
 import jadx.core.dex.nodes.FieldNode;
@@ -113,6 +115,10 @@ public class UsageInfo implements IUsageInfoData {
 			return;
 		}
 		if (type.isObject() && !type.isGenericType()) {
+			ClspClass clsDetails = root.getClsp().getClsDetails(type);
+			if (clsDetails != null && clsDetails.getSource() == ClspClassSource.APACHE_HTTP_LEGACY_CLIENT) {
+				root.getGradleInfoStorage().setUseApacheHttpLegacy(true);
+			}
 			ClassNode clsNode = root.resolveClass(type);
 			if (clsNode != null) {
 				consumer.accept(clsNode);

jadx-core/src/main/java/jadx/core/export/ExportGradleProject.java

@@ -80,6 +80,9 @@ public class ExportGradleProject {
 		if (gradleInfo.isVectorPathData() && minSdkVersion < 21 || gradleInfo.isVectorFillType() && minSdkVersion < 24) {
 			additionalOptions.add("vectorDrawables.useSupportLibrary = true");
 		}
+		if (gradleInfo.isUseApacheHttpLegacy()) {
+			additionalOptions.add("useLibrary 'org.apache.http.legacy'");
+		}
 		genAdditionalAndroidPluginOptions(tmpl, additionalOptions);
 
 		tmpl.save(new File(appDir, "build.gradle"));

jadx-core/src/main/java/jadx/core/export/GradleInfoStorage.java

@@ -6,6 +6,8 @@ public class GradleInfoStorage {
 
 	private boolean vectorFillType;
 
+	private boolean useApacheHttpLegacy;
+
 	public boolean isVectorPathData() {
 		return vectorPathData;
 	}
@@ -21,4 +23,12 @@ public class GradleInfoStorage {
 	public void setVectorFillType(boolean vectorFillType) {
 		this.vectorFillType = vectorFillType;
 	}
+
+	public boolean isUseApacheHttpLegacy() {
+		return useApacheHttpLegacy;
+	}
+
+	public void setUseApacheHttpLegacy(boolean useApacheHttpLegacy) {
+		this.useApacheHttpLegacy = useApacheHttpLegacy;
+	}
 }

jadx-core/src/test/java/jadx/tests/export/TestApacheHttpClient.java

@@ -0,0 +1,23 @@
+package jadx.tests.export;
+
+import org.junit.jupiter.api.Test;
+
+import jadx.core.export.GradleInfoStorage;
+import jadx.tests.api.ExportGradleTest;
+
+import static jadx.tests.api.utils.assertj.JadxAssertions.assertThat;
+
+public class TestApacheHttpClient extends ExportGradleTest {
+
+	@Test
+	void test() {
+		GradleInfoStorage gradleInfo = getRootNode().getGradleInfoStorage();
+		gradleInfo.setUseApacheHttpLegacy(true);
+		exportGradle("OptionalTargetSdkVersion.xml", "strings.xml");
+		assertThat(getAppGradleBuild()).contains("        useLibrary 'org.apache.http.legacy'");
+
+		gradleInfo.setUseApacheHttpLegacy(false);
+		exportGradle("OptionalTargetSdkVersion.xml", "strings.xml");
+		assertThat(getAppGradleBuild()).doesNotContain("        useLibrary 'org.apache.http.legacy'");
+	}
+}

jadx-core/src/test/java/jadx/tests/integration/others/TestUsageApacheHttpClient.java

@@ -0,0 +1,28 @@
+package jadx.tests.integration.others;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import jadx.core.dex.nodes.ClassNode;
+import jadx.tests.api.SmaliTest;
+
+public class TestUsageApacheHttpClient extends SmaliTest {
+
+	// @formatter:off
+	/*
+		package others;
+		import org.apache.http.client.HttpClient;
+
+		public class HttpClientTest {
+			private HttpClient httpClient;
+		}
+	*/
+	// @formatter:on
+
+	@Test
+	public void test() {
+		disableCompilation();
+		ClassNode cls = getClassNodeFromSmali();
+		Assertions.assertTrue(cls.root().getGradleInfoStorage().isUseApacheHttpLegacy());
+	}
+}

jadx-core/src/test/smali/others/TestUsageApacheHttpClient.smali

@@ -0,0 +1,17 @@
+###### Class Lothers.TestUsageApacheHttpClient (TestUsageApacheHttpClient)
+.class public Lothers/TestUsageApacheHttpClient;
+.super Ljava/lang/Object;
+.source "TestUsageApacheHttpClient.java"
+
+# instance fields
+.field private httpClient:Lorg/apache/http/client/HttpClient;
+
+# direct methods
+.method public constructor <init>()V
+    .registers 1
+
+    .line 3
+    invoke-direct {p0}, Ljava/lang/Object;-><init>()V
+
+    return-void
+.end method