在tinker支持androidx之后，最近把公司的项目迁移到了androidx，也用r8替换了proguard(r8真香，这里提一个坑，动态代理的类在proguard里默认是不会被混淆的，但是在r8里会，并且r8会非常积极的优化方法数)。然后最近一次合并代码后发现打包失败。失败原因是方法数超标。。(突然冒出了3w多个方法)。
没办法只能去跟一下D8下maindex的生成逻辑
首先去先确认一下build/intermediates/legacy_multidex_aapt_derived_proguard_rules 下的manifest_keep.txt文件，这个是用来扫描所有dexclass并找出rootclass的rules，这些rules遵循proguard的语法逻辑。
build/intermediates/legacy_multidex_main_dex_list 下mainDexList.txt则是最终产物，这些class会被放到main dex里。
接下来我们结合gradle plugin的源代码去跟一下具体实现。
首先找到D8MainDexListTransform这个类
override fun transform(invocation: TransformInvocation) {
logger.verbose("Generating the main dex list using D8.")
try {
val inputs = getByInputType(invocation)
val programFiles = inputs[ProguardInput.INPUT_JAR]!!
val libraryFiles = inputs[ProguardInput.LIBRARY_JAR]!! + bootClasspath.get()
logger.verbose("Program files: %s", programFiles.joinToString())
logger.verbose("Library files: %s", libraryFiles.joinToString())
logger.verbose(
"Proguard rule files: %s",
listOfNotNull(manifestProguardRules, userProguardRules).joinToString())
val proguardRules =
listOfNotNull(manifestProguardRules.singleFile().toPath(), userProguardRules)
val mainDexClasses = mutableSetOf<String>()
mainDexClasses.addAll(
//这里调用D8MainDexList.generate方法并返回计算出的所有classlist
D8MainDexList.generate(
getPlatformRules(),
proguardRules,
programFiles,
libraryFiles,
messageReceiver
)
)
if (userClasses != null) {
mainDexClasses.addAll(Files.readAllLines(userClasses))
}
Files.deleteIfExists(outputMainDexList)
Files.write(outputMainDexList, mainDexClasses)
} catch (e: D8MainDexList.MainDexListException) {
throw TransformException("Error while generating the main dex list:${System.lineSeparator()}${e.message}", e)
}
}
这边可以看到实际上运行的是D8MainDexList这个类的generate方法，并传入相关的rules跟files。然而在gradle plugin的源码里并没有这个类。google了一下果然在D8的项目里
private List<String> run(AndroidApp app, ExecutorService executor)
throws IOException {
try {
DexApplication application =
new ApplicationReader(app, options, timing).read(executor).toDirect();
AppView<? extends AppInfoWithSubtyping> appView =
AppView.createForR8(new AppInfoWithSubtyping(application), options);
appView.setAppServices(AppServices.builder(appView).build());
//这里通过rootsetbuilder 返回一个maindexrootset
RootSet mainDexRootSet =
new RootSetBuilder(appView, application, options.mainDexKeepRules).run(executor);
GraphConsumer graphConsumer = options.mainDexKeptGraphConsumer;
WhyAreYouKeepingConsumer whyAreYouKeepingConsumer = null;
if (!mainDexRootSet.reasonAsked.isEmpty()) {
whyAreYouKeepingConsumer = new WhyAreYouKeepingConsumer(graphConsumer);
graphConsumer = whyAreYouKeepingConsumer;
}
Enqueuer enqueuer = EnqueuerFactory.createForMainDexTracing(appView, graphConsumer);
//这里通过Enqueuer 把mainDexRootSet转换成 Set<DexType> liveTypes
Set<DexType> liveTypes = enqueuer.traceMainDex(mainDexRootSet, executor, timing);
// LiveTypes is the result.
//这里再根据liveTypes生成MainDexClasses。这个里面就是我们maindex里所有的类了。也就是 mainDexList.txt里的数据
MainDexClasses mainDexClasses = new MainDexListBuilder(liveTypes, application).run();
List<String> result =
mainDexClasses.getClasses().stream()
.map(c -> c.toSourceString().replace('.', '/') + ".class")
.sorted()
.collect(Collectors.toList());
if (options.mainDexListConsumer != null) {
options.mainDexListConsumer.accept(String.join("\n", result), options.reporter);
}
R8.processWhyAreYouKeepingAndCheckDiscarded(
mainDexRootSet,
() -> {
ArrayList<DexProgramClass> classes = new ArrayList<>();
// TODO(b/131668850): This is not a deterministic order!
mainDexClasses
.getClasses()
.forEach(
type -> {
DexClass clazz = appView.definitionFor(type);
assert clazz.isProgramClass();
classes.add(clazz.asProgramClass());
});
return classes;
},
whyAreYouKeepingConsumer,
appView,
enqueuer,
true,
options,
timing,
executor);
return result;
} catch (ExecutionException e) {
throw unwrapExecutionException(e);
}
}
这边先说一下生成maindexlist之后，同样是r8项目里的ApplicationReader类
private void readMainDexList(DexApplication.Builder<?> builder, ExecutorService executorService,
List<Future<?>> futures) {
if (inputApp.hasMainDexList()) {
futures.add(executorService.submit(() -> {
for (StringResource resource : inputApp.getMainDexListResources()) {
builder.addToMainDexList(MainDexList.parseList(resource, itemFactory));
}
这里就是往主dex里写class了
builder.addToMainDexList(
inputApp.getMainDexClasses()
.stream()
.map(clazz -> itemFactory.createType(DescriptorUtils.javaTypeToDescriptor(clazz)))
.collect(Collectors.toList()));
}));
}
}
这里参考一下D8的调用参数说明文档，--main-dex-list path指的就是刚生产的main dex list（D8内部的调用居然也是通过命令行来执行的）。
接下来让我们回到D8MainDexList类，看一下具体main dex的类是怎么计算出来的。
首先让我们看一下RootSetBuilder，这里会根据规则过滤所有的类生成mainDexRootSet
if (rule instanceof ProguardKeepRule) {
if (clazz.isNotProgramClass()) {
return;
}
switch (((ProguardKeepRule) rule).getType()) {
case KEEP_CLASS_MEMBERS:
// Members mentioned at -keepclassmembers always depend on their holder.
preconditionSupplier = ImmutableMap.of(definition -> true, clazz);
markMatchingVisibleMethods(
clazz, memberKeepRules, rule, preconditionSupplier, false, ifRule);
markMatchingVisibleFields(
clazz, memberKeepRules, rule, preconditionSupplier, false, ifRule);
break;
case KEEP_CLASSES_WITH_MEMBERS:
if (!allRulesSatisfied(memberKeepRules, clazz)) {
break;
}
// fallthrough;
case KEEP:
markClass(clazz, rule, ifRule);
preconditionSupplier = new HashMap<>();
if (ifRule != null) {
// Static members in -keep are pinned no matter what.
preconditionSupplier.put(DexDefinition::isStaticMember, null);
// Instance members may need to be kept even though the holder is not instantiated.
preconditionSupplier.put(definition -> !definition.isStaticMember(), clazz);
} else {
// Members mentioned at -keep should always be pinned as long as that -keep rule is
// not triggered conditionally.
preconditionSupplier.put((definition -> true), null);
}
markMatchingVisibleMethods(
clazz, memberKeepRules, rule, preconditionSupplier, false, ifRule);
markMatchingVisibleFields(
clazz, memberKeepRules, rule, preconditionSupplier, false, ifRule);
break;
case CONDITIONAL:
throw new Unreachable("-if rule will be evaluated separately, not here.");
}
return;
}
核心代码是遍历所有的类文件根据manifest_keep.txt文件的规则，找出所有的类然后放到RootSet里，这里大部分代码是application下的（我这边由于使用了tinker，tinker会自己在transform时修改keep文件添加自己的相关类）同时比较神奇的是，D8自己把所有的annotation也keep下来了也就是说哪怕你的annotation级别是source 也会当做runtime被keep下来，不知道D8为什么要这么做。
接下来我们看Enqueuer类，这里会对基于RootSet里的所有类做分析，核心是trace方法跟markTypeAsLive方法。这个类的代码及其复杂，这里我也只是看了个大概用来解决问题，有兴趣的可以自己下载D8源码查看。
首先Enqueuer .traceMainDex方法会把之前RootSet里所有类的构造方法标记为alive并构造一个action放到workList里（Enqueuer这个类除了用来构造maindex还用来处理其他逻辑，但是我们这边只分析main dex相关代码）
private void enqueueRootItems(Map<DexReference, Set<ProguardKeepRuleBase>> items) {
items.entrySet().forEach(this::enqueueRootItem);
}
private void trace(ExecutorService executorService, Timing timing) throws ExecutionException {
timing.begin("Grow the tree.");
try {
while (true) {
long numOfLiveItems = (long) liveTypes.size();
numOfLiveItems += (long) liveMethods.items.size();
numOfLiveItems += (long) liveFields.items.size();
while (!workList.isEmpty()) {
Action action = workList.poll();
switch (action.kind) {
case MARK_INSTANTIATED:
processNewlyInstantiatedClass((DexClass) action.target, action.reason);
break;
case MARK_REACHABLE_FIELD:
markInstanceFieldAsReachable((DexField) action.target, action.reason);
break;
case MARK_REACHABLE_VIRTUAL:
markVirtualMethodAsReachable((DexMethod) action.target, false, action.reason);
break;
case MARK_REACHABLE_INTERFACE:
markVirtualMethodAsReachable((DexMethod) action.target, true, action.reason);
break;
case MARK_REACHABLE_SUPER:
markSuperMethodAsReachable((DexMethod) action.target,
(DexEncodedMethod) action.context);
break;
case MARK_METHOD_KEPT:
markMethodAsKept((DexEncodedMethod) action.target, action.reason);
break;
case MARK_FIELD_KEPT:
markFieldAsKept((DexEncodedField) action.target, action.reason);
break;
case MARK_METHOD_LIVE:
processNewlyLiveMethod(((DexEncodedMethod) action.target), action.reason);
break;
default:
throw new IllegalArgumentException(action.kind.toString());
}
}
// Continue fix-point processing if -if rules are enabled by items that newly became live.
long numOfLiveItemsAfterProcessing = (long) liveTypes.size();
numOfLiveItemsAfterProcessing += (long) liveMethods.items.size();
numOfLiveItemsAfterProcessing += (long) liveFields.items.size();
if (numOfLiveItemsAfterProcessing > numOfLiveItems) {
RootSetBuilder consequentSetBuilder = new RootSetBuilder(appView, rootSet.ifRules);
IfRuleEvaluator ifRuleEvaluator =
consequentSetBuilder.getIfRuleEvaluator(
liveFields.getItems(),
liveMethods.getItems(),
liveTypes,
targetedMethods.getItems(),
executorService);
ConsequentRootSet consequentRootSet = ifRuleEvaluator.run();
// TODO(b/132600955): This modifies the root set. Should the consequent be persistent?
rootSet.addConsequentRootSet(consequentRootSet);
enqueueRootItems(consequentRootSet.noShrinking);
// TODO(b/132828740): Seems incorrect that the precondition is not always met here.
consequentRootSet.dependentNoShrinking.forEach(
(precondition, dependentItems) -> enqueueRootItems(dependentItems));
// Check for compatibility rules indicating that the holder must be implicitly kept.
if (forceProguardCompatibility) {
consequentRootSet.dependentKeepClassCompatRule.forEach(
(precondition, compatRules) -> {
assert precondition.isDexType();
DexClass preconditionHolder = appView.definitionFor(precondition.asDexType());
compatEnqueueHolderIfDependentNonStaticMember(preconditionHolder, compatRules);
});
}
if (!workList.isEmpty()) {
continue;
}
}
// Continue fix-point processing while there are additional work items to ensure items that
// are passed to Java reflections are traced.
if (!pendingReflectiveUses.isEmpty()) {
pendingReflectiveUses.forEach(this::handleReflectiveBehavior);
pendingReflectiveUses.clear();
}
if (!proguardCompatibilityWorkList.isEmpty()) {
workList.addAll(proguardCompatibilityWorkList);
proguardCompatibilityWorkList.clear();
}
if (!workList.isEmpty()) {
continue;
}
// Notify each analysis that a fixpoint has been reached, and give each analysis an
// opportunity to add items to the worklist.
analyses.forEach(analysis -> analysis.notifyFixpoint(this, workList));
if (!workList.isEmpty()) {
continue;
}
// Reached the fixpoint.
break;
}
if (Log.ENABLED) {
Set<DexEncodedMethod> allLive = Sets.newIdentityHashSet();
for (Entry<DexType, SetWithReason<DexEncodedMethod>> entry : reachableVirtualMethods
.entrySet()) {
allLive.addAll(entry.getValue().getItems());
}
Set<DexEncodedMethod> reachableNotLive = Sets.difference(allLive, liveMethods.getItems());
Log.debug(getClass(), "%s methods are reachable but not live", reachableNotLive.size());
Log.info(getClass(), "Only reachable: %s", reachableNotLive);
Set<DexType> liveButNotInstantiated =
Sets.difference(liveTypes, instantiatedTypes.getItems());
Log.debug(getClass(), "%s classes are live but not instantiated",
liveButNotInstantiated.size());
Log.info(getClass(), "Live but not instantiated: %s", liveButNotInstantiated);
SetView<DexEncodedMethod> targetedButNotLive = Sets
.difference(targetedMethods.getItems(), liveMethods.getItems());
Log.debug(getClass(), "%s methods are targeted but not live", targetedButNotLive.size());
Log.info(getClass(), "Targeted but not live: %s", targetedButNotLive);
}
assert liveTypes.stream().allMatch(DexType::isClassType);
assert instantiatedTypes.getItems().stream().allMatch(DexType::isClassType);
} finally {
timing.end();
}
unpinLambdaMethods();
}
private void markTypeAsLive(DexType type, ScopedDexMethodSet seen) {
type = type.toBaseType(appView.dexItemFactory());
if (!type.isClassType()) {
// Ignore primitive types.
return;
}
if (liveTypes.add(type)) {
if (Log.ENABLED) {
Log.verbose(getClass(), "Type `%s` has become live.", type);
}
DexClass holder = appView.definitionFor(type);
if (holder == null) {
reportMissingClass(type);
return;
}
for (DexType iface : holder.interfaces.values) {
markInterfaceTypeAsLiveViaInheritanceClause(iface);
}
if (holder.superType != null) {
ScopedDexMethodSet seenForSuper =
scopedMethodsForLiveTypes.computeIfAbsent(
holder.superType, ignore -> new ScopedDexMethodSet());
seen.setParent(seenForSuper);
DexClass holderSuper = appView.definitionFor(holder.superType);
if (holderSuper != null && holderSuper.isProgramClass()) {
registerType(holder.superType, KeepReason.reachableFromLiveType(type));
}
markTypeAsLive(holder.superType, seenForSuper);
if (holder.isNotProgramClass()) {
// Library classes may only extend other implement library classes.
ensureNotFromProgramOrThrow(holder.superType, type);
for (DexType iface : holder.interfaces.values) {
ensureNotFromProgramOrThrow(iface, type);
}
}
}
KeepReason reason = KeepReason.reachableFromLiveType(type);
// We cannot remove virtual methods defined earlier in the type hierarchy if it is widening
// access and is defined in an interface:
//
// public interface I {
// void clone();
// }
//
// class Model implements I {
// public void clone() { ... } <-- this cannot be removed
// }
//
// Any class loading of Model with Model.clone() removed will result in an illegal access
// error because their exists an existing implementation (here it is Object.clone()). This is
// only a problem in the DEX VM. We have to make this check no matter the output because
// CF libraries can be used by Android apps. See b/136698023 for more information.
holder
.virtualMethods()
.forEach(
m -> {
if (seen.addMethodIfMoreVisible(m)
== AddMethodIfMoreVisibleResult.ADDED_MORE_VISIBLE
&& holder.isProgramClass()
&& appView.appInfo().methodDefinedInInterfaces(m, holder.type)) {
markMethodAsTargeted(m, reason);
}
});
// We also need to add the corresponding <clinit> to the set of live methods, as otherwise
// static field initialization (and other class-load-time sideeffects) will not happen.
if (holder.isProgramClass() && holder.hasClassInitializer()) {
DexEncodedMethod clinit = holder.getClassInitializer();
if (clinit != null && clinit.getOptimizationInfo().mayHaveSideEffects()) {
assert clinit.method.holder == holder.type;
markDirectStaticOrConstructorMethodAsLive(clinit, reason);
}
}
if (holder.isProgramClass() && holder.isSerializable(appView)) {
enqueueFirstNonSerializableClassInitializer(holder, reason);
}
if (holder.isProgramClass()) {
if (!holder.annotations.isEmpty()) {
processAnnotations(holder, holder.annotations.annotations);
}
// If this type has deferred annotations, we have to process those now, too.
Set<DexAnnotation> annotations = deferredAnnotations.remove(type);
if (annotations != null && !annotations.isEmpty()) {
assert holder.accessFlags.isAnnotation();
assert annotations.stream().allMatch(a -> a.annotation.type == holder.type);
annotations.forEach(annotation -> handleAnnotation(holder, annotation));
}
} else {
assert !deferredAnnotations.containsKey(holder.type);
}
rootSet.forEachDependentStaticMember(holder, appView, this::enqueueDependentItem);
compatEnqueueHolderIfDependentNonStaticMember(
holder, rootSet.getDependentKeepClassCompatRule(holder.getType()));
}
}
最后是MainDexDirectReferenceTracer类
public void run(Set<DexType> roots) {
for (DexType type : roots) {
DexClass clazz = appInfo.definitionFor(type);
// Should only happen for library classes, which are filtered out.
assert clazz != null;
consumer.accept(type);
// Super and interfaces are live, no need to add them.
//对当前类的返回类型跟参数类型添加到输出里
traceAnnotationsDirectDependencies(clazz.annotations);
// 对当前类的所有变量的类型添加到输出里
clazz.forEachField(field -> consumer.accept(field.field.type));
// 对当前类的所有方法添加返回类型跟参数到输出里
clazz.forEachMethod(method -> {
traceMethodDirectDependencies(method.method, consumer);
// 对当前类的所有方法添加返回类型跟参数到输出里并且对当前method调用DirectReferencesCollector
method.registerCodeReferences(codeDirectReferenceCollector);
});
}
}
private void traceMethodDirectDependencies(DexMethod method, Consumer<DexType> consumer) {
DexProto proto = method.proto;
//添加该方法返回类型
consumer.accept(proto.returnType);
for (DexType parameterType : proto.parameters.values) {
//添加该方法所有的参数类型
consumer.accept(parameterType);
}
}
private class DirectReferencesCollector extends UseRegistry {
private DirectReferencesCollector(DexItemFactory factory) {
super(factory);
}
@Override
public boolean registerInvokeVirtual(DexMethod method) {
return registerInvoke(method);
}
@Override
public boolean registerInvokeDirect(DexMethod method) {
return registerInvoke(method);
}
@Override
public boolean registerInvokeStatic(DexMethod method) {
return registerInvoke(method);
}
@Override
public boolean registerInvokeInterface(DexMethod method) {
return registerInvoke(method);
}
@Override
public boolean registerInvokeSuper(DexMethod method) {
return registerInvoke(method);
}
protected boolean registerInvoke(DexMethod method) {
//添加所有方法内方法的持有者类型
consumer.accept(method.holder);
traceMethodDirectDependencies(method, consumer);
return true;
}
@Override
public boolean registerInstanceFieldWrite(DexField field) {
return registerFieldAccess(field);
}
@Override
public boolean registerInstanceFieldRead(DexField field) {
return registerFieldAccess(field);
}
@Override
public boolean registerStaticFieldRead(DexField field) {
return registerFieldAccess(field);
}
@Override
public boolean registerStaticFieldWrite(DexField field) {
return registerFieldAccess(field);
}
protected boolean registerFieldAccess(DexField field) {
//添加所有变量的类的类型跟持有该变量的类的类型
consumer.accept(field.holder);
consumer.accept(field.type);
return true;
}
@Override
public boolean registerNewInstance(DexType type) {
//添加所有实例的类
consumer.accept(type);
return true;
}
@Override
public boolean registerTypeReference(DexType type) {
//添加声明的所有类型
consumer.accept(type);
return true;
}
}
以上添加类型时会同时添加当前类与其supertype
最后就把以上结果输出到最开始提到的mainDexList.txt里让后续D8根据这个来拆分dex。
网友评论