From ecf432fadc3f56e18f863c68ee6ba6371ead92fd Mon Sep 17 00:00:00 2001 From: Federico Fissore Date: Thu, 30 Apr 2015 14:02:10 +0200 Subject: [PATCH] Boards Manager: downloading additional files as specified in preference 'boardsmanager.additional.urls' --- .../packages/ui/ContributionManagerUI.java | 2 + app/src/processing/app/Base.java | 3 +- .../contributions/packages/Constants.java | 23 ++++++ .../packages/ContributionInstaller.java | 74 +++++++++++++------ .../packages/ContributionsIndexer.java | 21 ++---- .../packages/PackageIndexFilenameFilter.java | 18 +++++ 6 files changed, 104 insertions(+), 37 deletions(-) create mode 100644 arduino-core/src/cc/arduino/contributions/packages/Constants.java create mode 100644 arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java diff --git a/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java b/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java index 5aae222be..ac4ded033 100644 --- a/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java +++ b/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java @@ -149,6 +149,8 @@ public class ContributionManagerUI extends InstallerJDialog { try { setProgressVisible(true, ""); installer.updateIndex(); + List downloadedPackageIndexFiles = installer.updateIndex(); + installer.deleteUnknownFiles(downloadedPackageIndexFiles); onIndexesUpdated(); } catch (Exception e) { throw new RuntimeException(e); diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index 0961ebde0..7db2bd2cd 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -322,7 +322,8 @@ public class Base { lastStatus = progress.getStatus(); } }; - installer.updateIndex(); + List downloadedPackageIndexFiles = installer.updateIndex(); + installer.deleteUnknownFiles(downloadedPackageIndexFiles); indexer.parseIndex(); indexer.syncWithFilesystem(getHardwareFolder()); diff --git a/arduino-core/src/cc/arduino/contributions/packages/Constants.java b/arduino-core/src/cc/arduino/contributions/packages/Constants.java new file mode 100644 index 000000000..1afbe0075 --- /dev/null +++ b/arduino-core/src/cc/arduino/contributions/packages/Constants.java @@ -0,0 +1,23 @@ +package cc.arduino.contributions.packages; + +import java.util.Arrays; +import java.util.List; + +public class Constants { + + public static final String DEFAULT_INDEX_FILE_NAME = "package_index.json"; + public static final List PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel"); + public static final String PACKAGE_INDEX_URL; + + public static final String PREFERENCES_BOARDS_MANAGER_ADDITIONAL_URLS = "boardsmanager.additional.urls"; + + static { + String extenalPackageIndexUrl = System.getProperty("PACKAGE_INDEX_URL"); + if (extenalPackageIndexUrl != null && !"".equals(extenalPackageIndexUrl)) { + PACKAGE_INDEX_URL = extenalPackageIndexUrl; + } else { + PACKAGE_INDEX_URL = "http://downloads.arduino.cc/packages/package_index.json"; + } + } + +} diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java index b648b63d8..2c72d98db 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java @@ -28,6 +28,7 @@ */ package cc.arduino.contributions.packages; +import cc.arduino.contributions.GPGDetachedSignatureVerifier; import cc.arduino.utils.ArchiveExtractor; import cc.arduino.utils.MultiStepProgress; import cc.arduino.utils.Progress; @@ -36,6 +37,8 @@ import com.google.common.collect.Collections2; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.Executor; import processing.app.BaseNoGui; +import processing.app.I18n; +import processing.app.PreferencesData; import processing.app.helpers.FileUtils; import processing.app.helpers.filefilters.OnlyDirs; import processing.app.tools.CollectStdOutStdErrExecutor; @@ -44,27 +47,13 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import static processing.app.I18n._; import static processing.app.I18n.format; public class ContributionInstaller { - private static final String PACKAGE_INDEX_URL; - - static { - String extenalPackageIndexUrl = System.getProperty("PACKAGE_INDEX_URL"); - if (extenalPackageIndexUrl != null && !"".equals(extenalPackageIndexUrl)) { - PACKAGE_INDEX_URL = extenalPackageIndexUrl; - } else { - PACKAGE_INDEX_URL = "http://downloads.arduino.cc/packages/package_index.json"; - } - } - private final ContributionsIndexer indexer; private final DownloadableContributionsDownloader downloader; @@ -239,22 +228,46 @@ public class ContributionInstaller { } public List updateIndex() throws Exception { - List errors = new LinkedList(); MultiStepProgress progress = new MultiStepProgress(1); - downloadIndex(progress, PACKAGE_INDEX_URL); - try { - downloadIndex(progress, PACKAGE_INDEX_URL + ".sig"); - } catch (Exception e) { - //ignore errors + List downloadedPackageIndexFilesAccumulator = new LinkedList(); + downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL); + + Set packageIndexURLs = new HashSet(); + String additionalURLs = PreferencesData.get(Constants.PREFERENCES_BOARDS_MANAGER_ADDITIONAL_URLS, ""); + if (!"".equals(additionalURLs)) { + packageIndexURLs.addAll(Arrays.asList(additionalURLs.split(","))); + } + + for (String packageIndexURL : packageIndexURLs) { + downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL); } progress.stepDone(); - return errors; + return downloadedPackageIndexFilesAccumulator; } - private void downloadIndex(MultiStepProgress progress, String packageIndexUrl) throws Exception { + private void downloadIndexAndSignature(MultiStepProgress progress, List downloadedPackagedIndexFilesAccumulator, String packageIndexUrl) throws Exception { + File packageIndex = downloadIndex(progress, packageIndexUrl); + downloadedPackagedIndexFilesAccumulator.add(packageIndex.getName()); + try { + File packageIndexSignature = downloadIndex(progress, packageIndexUrl + ".sig"); + boolean signatureVerified = new GPGDetachedSignatureVerifier().verify(packageIndex, packageIndexSignature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key")); + if (signatureVerified) { + downloadedPackagedIndexFilesAccumulator.add(packageIndexSignature.getName()); + } else { + downloadedPackagedIndexFilesAccumulator.remove(packageIndex.getName()); + packageIndex.delete(); + packageIndexSignature.delete(); + System.err.println(I18n.format(_("{0} file signature verification failed. File ignored."), packageIndexUrl)); + } + } catch (Exception e) { + //ignore errors + } + } + + private File downloadIndex(MultiStepProgress progress, String packageIndexUrl) throws Exception { String statusText = _("Downloading platforms index..."); URL url = new URL(packageIndexUrl); String[] urlPathParts = url.getFile().split("/"); @@ -269,9 +282,24 @@ public class ContributionInstaller { if (!tmpFile.renameTo(outputFile)) { throw new Exception("An error occurred while updating platforms index!"); } + + return outputFile; } protected void onProgress(Progress progress) { // Empty } + + public void deleteUnknownFiles(List downloadedPackageIndexFiles) { + File preferencesFolder = indexer.getIndexFile(".").getParentFile(); + File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)); + if (additionalPackageIndexFiles == null) { + return; + } + for (File additionalPackageIndexFile : additionalPackageIndexFiles) { + if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) { + additionalPackageIndexFile.delete(); + } + } + } } diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index fde637615..64d0a41ed 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -47,16 +47,16 @@ import processing.app.debug.TargetPlatform; import processing.app.debug.TargetPlatformException; import processing.app.helpers.PreferencesMap; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.*; import static processing.app.helpers.filefilters.OnlyDirs.ONLY_DIRS; public class ContributionsIndexer { - private static final String DEFAULT_INDEX_FILE_NAME = "package_index.json"; - private static final List PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel"); - private final File packagesFolder; private final File stagingFolder; private final File preferencesFolder; @@ -69,18 +69,13 @@ public class ContributionsIndexer { } public void parseIndex() throws Exception { - File defaultIndexFile = getIndexFile(DEFAULT_INDEX_FILE_NAME); + File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME); if (!isSigned(defaultIndexFile)) { - throw new SignatureVerificationFailedException(DEFAULT_INDEX_FILE_NAME); + throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME); } index = parseIndex(defaultIndexFile); - File[] indexFiles = preferencesFolder.listFiles(new FilenameFilter() { - @Override - public boolean accept(File file, String name) { - return !DEFAULT_INDEX_FILE_NAME.equals(name) && name.startsWith("package_") && name.endsWith("_index.json"); - } - }); + File[] indexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)); for (File indexFile : indexFiles) { ContributionsIndex contributionsIndex = parseIndex(indexFile); @@ -137,7 +132,7 @@ public class ContributionsIndexer { } private boolean mergeAllowed(ContributedPackage contributedPackage, File indexFile) { - return !PROTECTED_PACKAGE_NAMES.contains(contributedPackage.getName()) || isSigned(indexFile); + return !Constants.PROTECTED_PACKAGE_NAMES.contains(contributedPackage.getName()) || isSigned(indexFile); } private boolean isSigned(File indexFile) { diff --git a/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java b/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java new file mode 100644 index 000000000..22c777305 --- /dev/null +++ b/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java @@ -0,0 +1,18 @@ +package cc.arduino.contributions.packages; + +import java.io.File; +import java.io.FilenameFilter; + +public class PackageIndexFilenameFilter implements FilenameFilter { + + private final String defaultPackageIndexFileName; + + public PackageIndexFilenameFilter(String defaultPackageIndexFileName) { + this.defaultPackageIndexFileName = defaultPackageIndexFileName; + } + + @Override + public boolean accept(File file, String name) { + return !defaultPackageIndexFileName.equals(name) && name.startsWith("package_") && name.endsWith("_index.json"); + } +}