diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java deleted file mode 100644 index 5a3143d98c1cf7..00000000000000 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.SECONDS; - -import com.android.dex.Dex; -import com.android.dex.DexFormat; -import com.android.dx.command.dexer.DxContext; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.io.ByteStreams; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.devtools.build.android.AndroidOptionsUtils; -import com.google.devtools.build.android.Converters.CompatExistingPathConverter; -import com.google.devtools.build.android.Converters.CompatPathConverter; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -/** - * Tool used by Bazel as a replacement for Android's {@code dx} tool that assembles a single or, if - * allowed and necessary, multiple {@code .dex} files from a given archive of {@code .dex} and - * {@code .class} files. The tool merges the {@code .dex} files it encounters into a single file - * and additionally encodes any {@code .class} files it encounters. If multidex is allowed then the - * tool will generate multiple files subject to the {@code .dex} file format's limits on the number - * of methods and fields. - */ -class DexFileMerger { - - /** File name prefix of a {@code .dex} file automatically loaded in an archive. */ - private static final String DEX_PREFIX = "classes"; - - /** Commandline options. */ - @Parameters(separators = "= ") - public static class Options { - @Parameter( - names = {"--input", "-i"}, - converter = CompatExistingPathConverter.class, - description = - "Input archives with .dex files to merge. Inputs are processed in given order, so" - + " classes from later inputs will be added after earlier inputs. Duplicate" - + " classes are dropped.") - public List inputArchives = ImmutableList.of(); - - @Parameter( - names = {"--output", "-o"}, - converter = CompatPathConverter.class, - description = "Output archive to write.") - public Path outputArchive; - - @Parameter(names = "--multidex", description = "Allow more than one .dex file in the output.") - public MultidexStrategy multidexMode = MultidexStrategy.OFF; - - @Parameter( - names = "--main-dex-list", - converter = CompatExistingPathConverter.class, - description = "List of classes to be placed into \"main\" classes.dex file.") - public Path mainDexListFile; - - @Parameter( - names = "--minimal-main-dex", - arity = 1, - description = - "If true, *only* classes listed in --main_dex_list file are placed into \"main\" " - + "classes.dex file.") - public boolean minimalMainDex; - - @Parameter( - names = "--verbose", - arity = 1, - description = - "If true, print information about the merged files and resulting files to stdout.") - public boolean verbose; - - @Parameter( - names = "--max-bytes-wasted-per-file", - description = - "Limit on conservatively allocated but unused bytes per dex file, which can enable " - + "faster merging.") - public int wasteThresholdPerDex; - - // Undocumented dx option for testing multidex logic - @Parameter( - names = "--set-max-idx-number", - description = "Limit on fields and methods in a single dex file.") - public int maxNumberOfIdxPerDex = DexFormat.MAX_MEMBER_IDX; - - @Parameter( - names = "--forceJumbo", - arity = 1, - description = "Typically not needed flag intended to imitate dx's --forceJumbo.") - public boolean forceJumbo = false; // dx's default - - @Parameter(names = "--dex_prefix", description = "Dex file output prefix.") - public String dexPrefix = DEX_PREFIX; // dx's default - } - - public static void main(String[] args) throws Exception { - Options options = new Options(); - String[] preprocessedArgs = AndroidOptionsUtils.runArgFilePreprocessor(args); - String[] normalizedArgs = - AndroidOptionsUtils.normalizeBooleanOptions(options, preprocessedArgs); - JCommander.newBuilder().addObject(options).build().parse(normalizedArgs); - - buildMergedDexFiles(options); - } - - @VisibleForTesting - static void buildMergedDexFiles(Options options) throws IOException { - ListeningExecutorService executor; - checkArgument(!options.inputArchives.isEmpty(), "Need at least one --input"); - checkArgument( - options.mainDexListFile == null || options.inputArchives.size() == 1, - "--main-dex-list only supported with exactly one --input, use DexFileSplitter for more"); - if (options.multidexMode.isMultidexAllowed()) { - executor = createThreadPool(); - } else { - checkArgument( - options.mainDexListFile == null, - "--main-dex-list is only supported with multidex enabled, but mode is: %s", - options.multidexMode); - checkArgument( - !options.minimalMainDex, - "--minimal-main-dex is only supported with multidex enabled, but mode is: %s", - options.multidexMode); - // We'll only ever merge and write one dex file, so multi-threading is pointless. - executor = MoreExecutors.newDirectExecutorService(); - } - - ImmutableSet classesInMainDex = options.mainDexListFile != null - ? ImmutableSet.copyOf(Files.readAllLines(options.mainDexListFile, UTF_8)) - : null; - PrintStream originalStdOut = System.out; - try (DexFileAggregator out = createDexFileAggregator(options, executor)) { - if (!options.verbose) { - // com.android.dx.merge.DexMerger prints status information to System.out that we silence - // here unless it was explicitly requested. (It also prints debug info to DxContext.out, - // which we populate accordingly below.) - System.setOut(Dexing.nullout); - } - - LinkedHashSet seen = new LinkedHashSet<>(); - for (Path inputArchive : options.inputArchives) { - // Simply merge files from inputs in order. Doing that with a main dex list doesn't work, - // but we rule out more than one input with a main dex list above. - try (ZipFile zip = new ZipFile(inputArchive.toFile())) { - ArrayList dexFiles = filesToProcess(zip); - if (classesInMainDex == null) { - processDexFiles(zip, dexFiles, seen, out); - } else { - // To honor --main_dex_list make two passes: - // 1. process only the classes listed in the given file - // 2. process the remaining files - Predicate mainDexFilter = - ZipEntryPredicates.classFileFilter(classesInMainDex); - processDexFiles(zip, Iterables.filter(dexFiles, mainDexFilter), seen, out); - // Fail if main_dex_list is too big, following dx's example - checkState(out.getDexFilesWritten() == 0, "Too many classes listed in main dex list " - + "file %s, main dex capacity exceeded", options.mainDexListFile); - if (options.minimalMainDex) { - out.flush(); // Start new .dex file if requested - } - processDexFiles( - zip, Iterables.filter(dexFiles, Predicates.not(mainDexFilter)), seen, out); - } - } - } - } finally { - // Kill threads in the pool so we don't hang - MoreExecutors.shutdownAndAwaitTermination(executor, 1, SECONDS); - System.setOut(originalStdOut); - } - } - - /** - * Returns all .dex and .class files in the given zip. .class files are unexpected but we'll - * deal with them later. - */ - private static ArrayList filesToProcess(ZipFile zip) { - ArrayList result = Lists.newArrayList( - Iterators.filter( - Iterators.forEnumeration(zip.entries()), - Predicates.and( - Predicates.not(ZipEntryPredicates.isDirectory()), - ZipEntryPredicates.suffixes(".dex", ".class")))); - Collections.sort(result, ZipEntryComparator.LIKE_DX); - return result; - } - - private static void processDexFiles( - ZipFile zip, - Iterable filesToProcess, - LinkedHashSet seen, - DexFileAggregator out) - throws IOException { - for (ZipEntry entry : filesToProcess) { - String filename = entry.getName(); - checkState(filename.endsWith(".dex"), "Input shouldn't contain .class files: %s", filename); - if (!seen.add(filename)) { - continue; // pick first occurrence of each file to match how JVM treats dupes on classpath - } - try (InputStream content = zip.getInputStream(entry)) { - // We don't want to use the Dex(InputStream) constructor because it closes the stream, - // which will break the for loop, and it has its own bespoke way of reading the file into - // a byte buffer before effectively calling Dex(byte[]) anyway. - out.add(new Dex(ByteStreams.toByteArray(content))); - } - } - } - - private static DexFileAggregator createDexFileAggregator( - Options options, ListeningExecutorService executor) throws IOException { - String filePrefix = options.dexPrefix; - if (options.multidexMode == MultidexStrategy.GIVEN_SHARD) { - checkArgument(options.inputArchives.size() == 1, - "--multidex=given_shard requires exactly one --input"); - Pattern namingPattern = Pattern.compile("([0-9]+)\\..*"); - Matcher matcher = namingPattern.matcher(options.inputArchives.get(0).toFile().getName()); - checkArgument(matcher.matches(), - "expect input named .xxx.zip for --multidex=given_shard but got %s", - options.inputArchives.get(0).toFile().getName()); - int shard = Integer.parseInt(matcher.group(1)); - checkArgument(shard > 0, "expect positive N in input named .xxx.zip but got %s", shard); - if (shard > 1) { // first shard conventionally isn't numbered - filePrefix += shard; - } - } - return new DexFileAggregator( - new DxContext(options.verbose ? System.out : ByteStreams.nullOutputStream(), System.err), - new DexFileArchive( - new ZipOutputStream( - new BufferedOutputStream(Files.newOutputStream(options.outputArchive)))), - executor, - options.multidexMode, - options.forceJumbo, - options.maxNumberOfIdxPerDex, - options.wasteThresholdPerDex, - filePrefix); - } - - /** - * Creates an unbounded thread pool executor, which is appropriate here since the number of tasks - * we will add to the thread pool is at most dozens and some of them perform I/O (ie, may block). - */ - private static ListeningExecutorService createThreadPool() { - return MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - } - - private DexFileMerger() { - } -}