[FEATURE] Асинхронная загрузка файлов. Теперь нужны правки рантайма.

This commit is contained in:
zaxar163 2019-07-27 18:00:21 +03:00
parent 0031200679
commit f9315d9a3d

View file

@ -9,6 +9,13 @@
import java.net.URL; import java.net.URL;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
@ -19,8 +26,10 @@
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy; import org.apache.http.impl.client.LaxRedirectStrategy;
import pro.gravit.utils.helper.CommonHelper;
import pro.gravit.utils.helper.IOHelper; import pro.gravit.utils.helper.IOHelper;
import pro.gravit.utils.helper.LogHelper; import pro.gravit.utils.helper.LogHelper;
import pro.gravit.utils.helper.VerifyHelper;
public class ListDownloader { public class ListDownloader {
@FunctionalInterface @FunctionalInterface
@ -43,28 +52,48 @@ public DownloadTask(String apply, long size) {
} }
} }
private static final AtomicInteger COUNTER_THR = new AtomicInteger(0);
private static final ThreadFactory FACTORY = r -> CommonHelper.newThread("Downloader Thread #" + COUNTER_THR.incrementAndGet(), true, r);
public void download(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback) throws IOException, URISyntaxException { public void download(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback) throws IOException, URISyntaxException {
try (CloseableHttpClient httpclient = HttpClients.custom() try (CloseableHttpClient httpclient = HttpClients.custom()
.setRedirectStrategy(new LaxRedirectStrategy()) .setRedirectStrategy(new LaxRedirectStrategy())
.build()) { .build()) {
HttpGet get = null;
for (DownloadTask apply : applies) { for (DownloadTask apply : applies) {
URI u = new URL(base.concat(IOHelper.urlEncode(apply.apply).replace("%2F", "/"))).toURI(); URI u = new URL(base.concat(IOHelper.urlEncode(apply.apply).replace("%2F", "/"))).toURI();
callback.stateChanged(apply.apply, 0L, apply.size); callback.stateChanged(apply.apply, 0L, apply.size);
Path targetPath = dstDirFile.resolve(apply.apply); Path targetPath = dstDirFile.resolve(apply.apply);
LogHelper.debug("Download URL: %s to file %s dir: %s", u.toString(), targetPath.toAbsolutePath().toString(), dstDirFile.toAbsolutePath().toString()); LogHelper.debug("Download URL: %s to file %s dir: %s", u.toString(), targetPath.toAbsolutePath().toString(), dstDirFile.toAbsolutePath().toString());
if (get == null) get = new HttpGet(u); HttpGet get = new HttpGet(u);
else { List<IOException> excs = new CopyOnWriteArrayList<>();
get.reset(); ExecutorService executor = newExecutor();
get.setURI(u); executor.submit(() -> {
try {
httpclient.execute(get, new FileDownloadResponseHandler(targetPath, apply, callback, totalCallback, false));
} catch (IOException e) {
excs.add(e);
}
});
executor.shutdown();
try {
executor.awaitTermination(24, TimeUnit.HOURS);
} catch (InterruptedException e) {
LogHelper.error(e);
}
if (!excs.isEmpty()) {
IOException toThrow = excs.remove(0);
excs.forEach(toThrow::addSuppressed);
throw toThrow;
} }
httpclient.execute(get, new FileDownloadResponseHandler(targetPath, apply, callback, totalCallback, false));
} }
} }
} }
public void downloadZip(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback, boolean fullDownload) throws IOException, URISyntaxException { private ExecutorService newExecutor() {
return new ThreadPoolExecutor(0, VerifyHelper.verifyInt(Integer.parseInt(System.getProperty("launcher.downloadThreads", "3")), VerifyHelper.POSITIVE, "Thread max count must be positive."), 5, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), FACTORY);
}
public void downloadZip(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback, boolean fullDownload) throws IOException, URISyntaxException {
/*try (CloseableHttpClient httpclient = HttpClients.custom() /*try (CloseableHttpClient httpclient = HttpClients.custom()
.setRedirectStrategy(new LaxRedirectStrategy()) .setRedirectStrategy(new LaxRedirectStrategy())
.build()) { .build()) {
@ -182,7 +211,6 @@ public static void transfer(InputStream input, Path file, String filename, long
try (OutputStream fileOutput = IOHelper.newOutput(file)) { try (OutputStream fileOutput = IOHelper.newOutput(file)) {
long downloaded = 0L; long downloaded = 0L;
// Download with digest update
byte[] bytes = IOHelper.newBuffer(); byte[] bytes = IOHelper.newBuffer();
while (downloaded < size) { while (downloaded < size) {
int remaining = (int) Math.min(size - downloaded, bytes.length); int remaining = (int) Math.min(size - downloaded, bytes.length);
@ -195,7 +223,6 @@ public static void transfer(InputStream input, Path file, String filename, long
// Update state // Update state
downloaded += length; downloaded += length;
//totalDownloaded += length;
totalCallback.addTotal(length); totalCallback.addTotal(length);
callback.stateChanged(filename, downloaded, size); callback.stateChanged(filename, downloaded, size);
} }