Boards Manager: downloading additional files as specified in preference 'boardsmanager.additional.urls'

app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java

@@ -149,6 +149,8 @@ public class ContributionManagerUI extends InstallerJDialog {
         try {
           setProgressVisible(true, "");
           installer.updateIndex();
+          List<String> downloadedPackageIndexFiles = installer.updateIndex();
+          installer.deleteUnknownFiles(downloadedPackageIndexFiles);
           onIndexesUpdated();
         } catch (Exception e) {
           throw new RuntimeException(e);

app/src/processing/app/Base.java

@@ -322,7 +322,8 @@ public class Base {
           lastStatus = progress.getStatus();
         }
       };
-      installer.updateIndex();
+      List<String> downloadedPackageIndexFiles = installer.updateIndex();
+      installer.deleteUnknownFiles(downloadedPackageIndexFiles);
       indexer.parseIndex();
       indexer.syncWithFilesystem(getHardwareFolder());
 

arduino-core/src/cc/arduino/contributions/packages/Constants.java

@@ -0,0 +1,23 @@
+package cc.arduino.contributions.packages;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class Constants {
+
+  public static final String DEFAULT_INDEX_FILE_NAME = "package_index.json";
+  public static final List<String> PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel");
+  public static final String PACKAGE_INDEX_URL;
+
+  public static final String PREFERENCES_BOARDS_MANAGER_ADDITIONAL_URLS = "boardsmanager.additional.urls";
+
+  static {
+    String extenalPackageIndexUrl = System.getProperty("PACKAGE_INDEX_URL");
+    if (extenalPackageIndexUrl != null && !"".equals(extenalPackageIndexUrl)) {
+      PACKAGE_INDEX_URL = extenalPackageIndexUrl;
+    } else {
+      PACKAGE_INDEX_URL = "http://downloads.arduino.cc/packages/package_index.json";
+    }
+  }
+
+}

arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java

@@ -28,6 +28,7 @@
  */
 package cc.arduino.contributions.packages;
 
+import cc.arduino.contributions.GPGDetachedSignatureVerifier;
 import cc.arduino.utils.ArchiveExtractor;
 import cc.arduino.utils.MultiStepProgress;
 import cc.arduino.utils.Progress;
@@ -36,6 +37,8 @@ import com.google.common.collect.Collections2;
 import org.apache.commons.exec.CommandLine;
 import org.apache.commons.exec.Executor;
 import processing.app.BaseNoGui;
+import processing.app.I18n;
+import processing.app.PreferencesData;
 import processing.app.helpers.FileUtils;
 import processing.app.helpers.filefilters.OnlyDirs;
 import processing.app.tools.CollectStdOutStdErrExecutor;
@@ -44,27 +47,13 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
-import java.util.Collection;
+import java.util.*;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
 
 import static processing.app.I18n._;
 import static processing.app.I18n.format;
 
 public class ContributionInstaller {
 
-  private static final String PACKAGE_INDEX_URL;
-
-  static {
-    String extenalPackageIndexUrl = System.getProperty("PACKAGE_INDEX_URL");
-    if (extenalPackageIndexUrl != null && !"".equals(extenalPackageIndexUrl)) {
-      PACKAGE_INDEX_URL = extenalPackageIndexUrl;
-    } else {
-      PACKAGE_INDEX_URL = "http://downloads.arduino.cc/packages/package_index.json";
-    }
-  }
-
   private final ContributionsIndexer indexer;
   private final DownloadableContributionsDownloader downloader;
 
@@ -239,22 +228,46 @@ public class ContributionInstaller {
   }
 
   public List<String> updateIndex() throws Exception {
-    List<String> errors = new LinkedList<String>();
     MultiStepProgress progress = new MultiStepProgress(1);
 
-    downloadIndex(progress, PACKAGE_INDEX_URL);
+    List<String> downloadedPackageIndexFilesAccumulator = new LinkedList<String>();
-    try {
+    downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL);
-      downloadIndex(progress, PACKAGE_INDEX_URL + ".sig");
+
-    } catch (Exception e) {
+    Set<String> packageIndexURLs = new HashSet<String>();
-      //ignore errors
+    String additionalURLs = PreferencesData.get(Constants.PREFERENCES_BOARDS_MANAGER_ADDITIONAL_URLS, "");
+    if (!"".equals(additionalURLs)) {
+      packageIndexURLs.addAll(Arrays.asList(additionalURLs.split(",")));
+    }
+
+    for (String packageIndexURL : packageIndexURLs) {
+      downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL);
     }
 
     progress.stepDone();
 
-    return errors;
+    return downloadedPackageIndexFilesAccumulator;
   }
 
-  private void downloadIndex(MultiStepProgress progress, String packageIndexUrl) throws Exception {
+  private void downloadIndexAndSignature(MultiStepProgress progress, List<String> downloadedPackagedIndexFilesAccumulator, String packageIndexUrl) throws Exception {
+    File packageIndex = downloadIndex(progress, packageIndexUrl);
+    downloadedPackagedIndexFilesAccumulator.add(packageIndex.getName());
+    try {
+      File packageIndexSignature = downloadIndex(progress, packageIndexUrl + ".sig");
+      boolean signatureVerified = new GPGDetachedSignatureVerifier().verify(packageIndex, packageIndexSignature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key"));
+      if (signatureVerified) {
+        downloadedPackagedIndexFilesAccumulator.add(packageIndexSignature.getName());
+      } else {
+        downloadedPackagedIndexFilesAccumulator.remove(packageIndex.getName());
+        packageIndex.delete();
+        packageIndexSignature.delete();
+        System.err.println(I18n.format(_("{0} file signature verification failed. File ignored."), packageIndexUrl));
+      }
+    } catch (Exception e) {
+      //ignore errors
+    }
+  }
+
+  private File downloadIndex(MultiStepProgress progress, String packageIndexUrl) throws Exception {
     String statusText = _("Downloading platforms index...");
     URL url = new URL(packageIndexUrl);
     String[] urlPathParts = url.getFile().split("/");
@@ -269,9 +282,24 @@ public class ContributionInstaller {
     if (!tmpFile.renameTo(outputFile)) {
       throw new Exception("An error occurred while updating platforms index!");
     }
+
+    return outputFile;
   }
 
   protected void onProgress(Progress progress) {
     // Empty
   }
+
+  public void deleteUnknownFiles(List<String> downloadedPackageIndexFiles) {
+    File preferencesFolder = indexer.getIndexFile(".").getParentFile();
+    File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME));
+    if (additionalPackageIndexFiles == null) {
+      return;
+    }
+    for (File additionalPackageIndexFile : additionalPackageIndexFiles) {
+      if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) {
+        additionalPackageIndexFile.delete();
+      }
+    }
+  }
 }

arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java

@@ -47,16 +47,16 @@ import processing.app.debug.TargetPlatform;
 import processing.app.debug.TargetPlatformException;
 import processing.app.helpers.PreferencesMap;
 
-import java.io.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.*;
 
 import static processing.app.helpers.filefilters.OnlyDirs.ONLY_DIRS;
 
 public class ContributionsIndexer {
 
-  private static final String DEFAULT_INDEX_FILE_NAME = "package_index.json";
-  private static final List<String> PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel");
-
   private final File packagesFolder;
   private final File stagingFolder;
   private final File preferencesFolder;
@@ -69,18 +69,13 @@ public class ContributionsIndexer {
   }
 
   public void parseIndex() throws Exception {
-    File defaultIndexFile = getIndexFile(DEFAULT_INDEX_FILE_NAME);
+    File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME);
     if (!isSigned(defaultIndexFile)) {
-      throw new SignatureVerificationFailedException(DEFAULT_INDEX_FILE_NAME);
+      throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME);
     }
     index = parseIndex(defaultIndexFile);
 
-    File[] indexFiles = preferencesFolder.listFiles(new FilenameFilter() {
+    File[] indexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME));
-      @Override
-      public boolean accept(File file, String name) {
-        return !DEFAULT_INDEX_FILE_NAME.equals(name) && name.startsWith("package_") && name.endsWith("_index.json");
-      }
-    });
 
     for (File indexFile : indexFiles) {
       ContributionsIndex contributionsIndex = parseIndex(indexFile);
@@ -137,7 +132,7 @@ public class ContributionsIndexer {
   }
 
   private boolean mergeAllowed(ContributedPackage contributedPackage, File indexFile) {
-    return !PROTECTED_PACKAGE_NAMES.contains(contributedPackage.getName()) || isSigned(indexFile);
+    return !Constants.PROTECTED_PACKAGE_NAMES.contains(contributedPackage.getName()) || isSigned(indexFile);
   }
 
   private boolean isSigned(File indexFile) {

arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java

@@ -0,0 +1,18 @@
+package cc.arduino.contributions.packages;
+
+import java.io.File;
+import java.io.FilenameFilter;
+
+public class PackageIndexFilenameFilter implements FilenameFilter {
+
+  private final String defaultPackageIndexFileName;
+
+  public PackageIndexFilenameFilter(String defaultPackageIndexFileName) {
+    this.defaultPackageIndexFileName = defaultPackageIndexFileName;
+  }
+
+  @Override
+  public boolean accept(File file, String name) {
+    return !defaultPackageIndexFileName.equals(name) && name.startsWith("package_") && name.endsWith("_index.json");
+  }
+}