RAT-322: be able to scan hidden directories

apache · Nov 25, 2023 · 8d6b661 · 8d6b661
1 parent c2e51ad
commit 8d6b661
Show file tree

Hide file tree

Showing 13 changed files with 316 additions and 35 deletions.
diff --git a/apache-rat-core/src/main/java/org/apache/rat/Report.java b/apache-rat-core/src/main/java/org/apache/rat/Report.java
@@ -39,6 +39,7 @@
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.commons.io.filefilter.NameFileFilter;
 import org.apache.commons.io.filefilter.NotFileFilter;
 import org.apache.commons.io.filefilter.OrFileFilter;
@@ -95,6 +96,12 @@ public class Report {
      * Do not use the default files.
      */
     private static final String NO_DEFAULTS = "no-default-licenses";
+
+    /**
+     * Scan hidden directories
+     */
+    private static final String SCAN_HIDDEN_DIRECTORIES = "scan-hidden-directories";
+
     /**
      * List the licenses that were used for the run.
      */
@@ -162,6 +169,10 @@ public static final void main(String[] args) throws Exception {
     static ReportConfiguration createConfiguration(String baseDirectory, CommandLine cl) throws IOException {
         final ReportConfiguration configuration = new ReportConfiguration();
 
+        if (cl.hasOption(SCAN_HIDDEN_DIRECTORIES)) {
+            configuration.setDirectoryFilter(null);
+        }
+
         if (cl.hasOption('a') || cl.hasOption('A')) {
             configuration.setAddLicenseHeaders(cl.hasOption('f') ? AddLicenseHeaders.FORCED : AddLicenseHeaders.TRUE);
             configuration.setCopyrightMessage(cl.getOptionValue("c"));
@@ -268,6 +279,7 @@ static Options buildOptions() {
         opts.addOption(null, LICENSES, true, "File names or URLs for license definitions");
         opts.addOption(null, LIST_LICENSES, false, "List all active licenses");
         opts.addOption(null, LIST_LICENSE_FAMILIES, false, "List all defined license families");
+        opts.addOption(null, SCAN_HIDDEN_DIRECTORIES, false, "Scan hidden directories");
 
         OptionGroup addLicenseGroup = new OptionGroup();
         String addLicenseDesc = "Add the default license header to any file with an unknown license that is not in the exclusion list. "
@@ -356,7 +368,7 @@ private static IReportable getDirectory(String baseDirectory, ReportConfiguratio
             }
 
             if (base.isDirectory()) {
-                return new DirectoryWalker(base, config.getInputFileFilter());
+                return new DirectoryWalker(base, config.getInputFileFilter(), config.getDirectoryFilter());
             }
 
             try {

diff --git a/apache-rat-core/src/main/java/org/apache/rat/ReportConfiguration.java b/apache-rat-core/src/main/java/org/apache/rat/ReportConfiguration.java
@@ -37,6 +37,9 @@
 import java.util.TreeSet;
 import java.util.function.Consumer;
 
+import org.apache.commons.io.filefilter.FalseFileFilter;
+import org.apache.commons.io.filefilter.HiddenFileFilter;
+import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.io.function.IOSupplier;
 import org.apache.rat.config.AddLicenseHeaders;
 import org.apache.rat.license.ILicense;
@@ -45,6 +48,7 @@
 import org.apache.rat.license.LicenseSetFactory;
 import org.apache.rat.license.LicenseSetFactory.LicenseFilter;
 import org.apache.rat.report.IReportable;
+import org.apache.rat.walker.NameBasedHiddenFileFilter;
 
 /**
  * A configuration object is used by the front end to invoke the
@@ -64,6 +68,7 @@ public class ReportConfiguration {
     private IOSupplier<InputStream> styleSheet = null;
     private IReportable reportable = null;
     private FilenameFilter inputFileFilter = null;
+    private IOFileFilter directoryFilter = NameBasedHiddenFileFilter.HIDDEN;
 
     /**
      * @return The filename filter for the potential input files.
@@ -79,6 +84,22 @@ public void setInputFileFilter(FilenameFilter inputFileFilter) {
         this.inputFileFilter = inputFileFilter;
     }
 
+    public IOFileFilter getDirectoryFilter() {
+        return directoryFilter;
+    }
+
+    public void setDirectoryFilter(IOFileFilter directoryFilter) {
+        if (directoryFilter == null) {
+            this.directoryFilter = FalseFileFilter.FALSE;
+        } else {
+            this.directoryFilter = directoryFilter;
+        }
+    }
+
+    public void addDirectoryFilter(IOFileFilter directoryFilter) {
+        this.directoryFilter = this.directoryFilter.and(directoryFilter);
+    }
+
     /**
      * @return the thing being reported on.
      */

diff --git a/apache-rat-core/src/main/java/org/apache/rat/walker/DirectoryWalker.java b/apache-rat-core/src/main/java/org/apache/rat/walker/DirectoryWalker.java
@@ -19,6 +19,8 @@
 
 package org.apache.rat.walker;
 
+import org.apache.commons.io.filefilter.HiddenFileFilter;
+import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.rat.api.Document;
 import org.apache.rat.api.RatException;
 import org.apache.rat.document.impl.FileDocument;
@@ -37,27 +39,41 @@ public class DirectoryWalker extends Walker implements IReportable {
 
     protected static final FileNameComparator COMPARATOR = new FileNameComparator();
 
-    public DirectoryWalker(File file) {
-        this(file, (FilenameFilter) null);
+    private IOFileFilter directoryFilter;
+
+    /**
+     * Constructs a walker.
+     *
+     * @param file the directory to walk.
+     * @param directoryFilter directory filter to eventually exclude some directories/files from the scan.
+     */
+    public DirectoryWalker(File file, IOFileFilter directoryFilter) {
+        this(file, (FilenameFilter) null, directoryFilter);
     }
 
     /**
      * Constructs a walker.
      *
-     * @param file   not null
+     * @param file the directory to walk (not null).
      * @param filter filters input files (optional),
      *               or null when no filtering should be performed
+     * @param directoryFilter filters directories (optional), or null when no filtering should be performed.
      */
-    public DirectoryWalker(File file, final FilenameFilter filter) {
+    public DirectoryWalker(File file, final FilenameFilter filter, IOFileFilter directoryFilter) {
         super(file.getPath(), file, filter);
+        this.directoryFilter = directoryFilter;
     }
 
-    public DirectoryWalker(File file, final Pattern ignoreNameRegex) {
+    /**
+     * Constructs a walker.
+     *
+     * @param file the directory to walk (not null).
+     * @param ignoreNameRegex ignore directories/files with name matching the regex.
+     * @param directoryFilter filters directories (optional), or null when no filtering should be performed.
+     */
+    public DirectoryWalker(File file, final Pattern ignoreNameRegex, IOFileFilter directoryFilter) {
         super(file.getPath(), file, regexFilter(ignoreNameRegex));
-    }
-
-    public boolean isRestricted() {
-        return false;
+        this.directoryFilter = directoryFilter;
     }
 
     /**
@@ -68,7 +84,11 @@ public boolean isRestricted() {
      * @throws RatException
      */
     private void processDirectory(RatReport report, final File file) throws RatException {
-        if (!isRestricted(file)) {
+        if (directoryFilter != null) {
+            if (!directoryFilter.accept(file)) {
+                process(report, file);
+            }
+        } else {
             process(report, file);
         }
     }

diff --git a/apache-rat-core/src/main/java/org/apache/rat/walker/NameBasedHiddenFileFilter.java b/apache-rat-core/src/main/java/org/apache/rat/walker/NameBasedHiddenFileFilter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ */
+package org.apache.rat.walker;
+
+import org.apache.commons.io.filefilter.AbstractFileFilter;
+import org.apache.commons.io.filefilter.IOFileFilter;
+
+import java.io.File;
+import java.io.Serializable;
+
+/**
+ * This filter accepts Files that are hidden, e.g. file name starts with .
+ *
+ * Example, showing how to print out a list of the current directory's hidden files:
+ *
+ * Using Classic IO
+ *   File dir = new File(".");
+ *   String[] files = dir.list(NameBasedHiddenFileFilter.HIDDEN);
+ *   for (String file : files) {
+ *       System.out.println(file);
+ *   }
+ *
+ * Using NIO
+ *
+ *   final Path dir = Paths.get("");
+ *   final AccumulatorPathVisitor visitor = AccumulatorPathVisitor.withLongCounters(NameBasedHiddenFileFilter.HIDDEN);
+ *   //
+ *   // Walk one dir
+ *   Files.walkFileTree(dir, Collections.emptySet(), 1, visitor);
+ *   System.out.println(visitor.getPathCounters());
+ *   System.out.println(visitor.getFileList());
+ *   //
+ *   visitor.getPathCounters().reset();
+ *   //
+ *   // Walk dir tree
+ *   Files.walkFileTree(dir, visitor);
+ *   System.out.println(visitor.getPathCounters());
+ *   System.out.println(visitor.getDirList());
+ *   System.out.println(visitor.getFileList());
+ */
+public class NameBasedHiddenFileFilter extends AbstractFileFilter implements Serializable {
+
+    /**
+     * Singleton instance of hidden filter.
+     */
+    public static final IOFileFilter HIDDEN = new NameBasedHiddenFileFilter();
+
+    /**
+     * Restrictive constructor.
+     */
+    protected NameBasedHiddenFileFilter() {
+    }
+
+    /**
+     * Checks to see if the file is hidden, e.g. file name starts with .
+     *
+     * @param file the File to check
+     * @return true if the file is hidden (file name starting with .), false else
+     */
+    @Override
+    public boolean accept(File file) {
+        if (file.getName().startsWith(".")) {
+            return true;
+        }
+        return false;
+    }
+
+}
diff --git a/apache-rat-core/src/main/java/org/apache/rat/walker/Walker.java b/apache-rat-core/src/main/java/org/apache/rat/walker/Walker.java
@@ -48,10 +48,6 @@ public boolean accept(File dir, String name) {
             }
         };
     }
-
-    protected boolean isRestricted(File file) {
-        return file.getName().startsWith(".");
-    }
 
     protected final boolean isNotIgnored(final File file) {
         boolean result = false;

diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReportConfigurationTest.java b/apache-rat-core/src/test/java/org/apache/rat/ReportConfigurationTest.java
@@ -43,6 +43,10 @@
 import java.util.List;
 import java.util.SortedSet;
 
+import org.apache.commons.io.filefilter.AndFileFilter;
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
+import org.apache.commons.io.filefilter.FalseFileFilter;
+import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.commons.io.function.IOSupplier;
 import org.apache.rat.ReportConfiguration.NoCloseOutputStream;
 import org.apache.rat.config.AddLicenseHeaders;
@@ -52,6 +56,7 @@
 import org.apache.rat.license.LicenseSetFactory.LicenseFilter;
 import org.apache.rat.report.IReportable;
 import org.apache.rat.testhelpers.TestingLicense;
+import org.apache.rat.walker.NameBasedHiddenFileFilter;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -189,6 +194,19 @@ public void inputFileFilterTest() {
         assertEquals(filter, underTest.getInputFileFilter());
     }
 
+    @Test
+    public void directoryFilterTest() {
+        assertNotNull("Directory filter should not be null by default", underTest.getDirectoryFilter());
+        assertTrue("Directory filter should be a NameBasedHiddenFileFilter one by default", underTest.getDirectoryFilter() instanceof NameBasedHiddenFileFilter);
+
+        underTest.setDirectoryFilter(DirectoryFileFilter.DIRECTORY);
+        underTest.addDirectoryFilter(NameBasedHiddenFileFilter.HIDDEN);
+        assertTrue("Using addDirectoryFilter() method should create a AndFileFilter", underTest.getDirectoryFilter() instanceof AndFileFilter);
+
+        underTest.setDirectoryFilter(null);
+        assertTrue("When setting directory filter to null, a FalseFileFilter should be created", underTest.getDirectoryFilter() instanceof FalseFileFilter);
+    }
+
     @Test
     public void licenseFamiliesTest() {
         assertTrue(underTest.getLicenseFamilies(LicenseFilter.all).isEmpty());
@@ -427,6 +445,8 @@ public static void validateDefault(ReportConfiguration config) {
         assertNull("Input file filter should be null", config.getInputFileFilter());
         assertTrue("Style report should be true", config.isStyleReport());
         assertNotNull("Stylesheet should not be null", config.getStyleSheet());
+        assertNotNull("Directory filter should not be null", config.getDirectoryFilter());
+        assertTrue("Directory filter should be hidden filter by default", config.getDirectoryFilter() instanceof NameBasedHiddenFileFilter);
 
         validateDefaultApprovedLicenses(config);
         validateDefaultLicenseFamilies(config);

diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java b/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java
@@ -30,6 +30,7 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathFactory;
 
+import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.rat.test.utils.Resources;
 import org.apache.rat.testhelpers.XmlUtils;
 import org.apache.rat.walker.DirectoryWalker;
@@ -51,7 +52,7 @@ public void xmlReportTest() throws Exception {
         final ReportConfiguration configuration = new ReportConfiguration();
         configuration.setStyleReport(false);
         configuration.setFrom(defaults);
-        configuration.setReportable(new DirectoryWalker(new File(elementsPath)));
+        configuration.setReportable(new DirectoryWalker(new File(elementsPath), HiddenFileFilter.HIDDEN));
         configuration.setOut(() -> out);
         Reporter.report(configuration);
         Document doc = XmlUtils.toDom(new ByteArrayInputStream(out.toByteArray()));
@@ -91,7 +92,7 @@ public void plainReportWithArchivesAndUnapprovedLicenses() throws Exception {
         final String elementsPath = Resources.getResourceDirectory("elements/Source.java");
         final ReportConfiguration configuration = new ReportConfiguration();
         configuration.setFrom(defaults);
-        configuration.setReportable(new DirectoryWalker(new File(elementsPath)));
+        configuration.setReportable(new DirectoryWalker(new File(elementsPath), HiddenFileFilter.HIDDEN));
         configuration.setOut(() -> out);
         Reporter.report(configuration);
 

diff --git a/apache-rat-core/src/test/java/org/apache/rat/report/xml/XmlReportFactoryTest.java b/apache-rat-core/src/test/java/org/apache/rat/report/xml/XmlReportFactoryTest.java
@@ -29,6 +29,7 @@
 import java.io.StringWriter;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.rat.ConfigurationException;
 import org.apache.rat.ReportConfiguration;
 import org.apache.rat.analysis.IHeaderMatcher.State;
@@ -76,7 +77,7 @@ public void standardReport() throws Exception {
 
         final TestingLicense testingLicense = new TestingLicense(new TestingMatcher(true), family);
 
-        DirectoryWalker directory = new DirectoryWalker(new File(elementsPath), IGNORE_EMPTY);
+        DirectoryWalker directory = new DirectoryWalker(new File(elementsPath), IGNORE_EMPTY, HiddenFileFilter.HIDDEN);
         final ClaimStatistic statistic = new ClaimStatistic();
         final ReportConfiguration configuration = new ReportConfiguration();
         configuration.addLicense(testingLicense);

diff --git a/apache-rat-core/src/test/java/org/apache/rat/report/xml/XmlReportTest.java b/apache-rat-core/src/test/java/org/apache/rat/report/xml/XmlReportTest.java
@@ -33,6 +33,7 @@
 import javax.xml.xpath.XPath;
 import javax.xml.xpath.XPathFactory;
 
+import org.apache.commons.io.filefilter.HiddenFileFilter;
 import org.apache.rat.analysis.DefaultAnalyserFactory;
 import org.apache.rat.analysis.IHeaderMatcher;
 import org.apache.rat.analysis.matchers.CopyrightMatcher;
@@ -88,7 +89,7 @@ private void report(DirectoryWalker directory) throws Exception {
     @Test
     public void baseReport() throws Exception {
         final String elementsPath = Resources.getResourceDirectory("elements/Source.java");
-        DirectoryWalker directory = new DirectoryWalker(new File(elementsPath), IGNORE);
+        DirectoryWalker directory = new DirectoryWalker(new File(elementsPath), IGNORE, HiddenFileFilter.HIDDEN);
         report.startReport();
         report(directory);
         report.endReport();