Lanmikeman/Launcher
1
1
Fork 0
mirror of https://github.com/GravitLauncher/Launcher synced 2024-11-15 03:31:15 +03:00
Code Issues Projects Releases Packages Wiki Activity

[FEATURE] Удаление ListDownloader

Browse source
This commit is contained in:
Gravit 2019-12-04 23:08:36 +07:00
parent 3296d00249
commit 03471e8bbf
No known key found for this signature in database
GPG key ID: 061981E1E85D3216
6 changed files with 13 additions and 561 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Launcher/src/main/java/pro/gravit/launcher/LauncherEngine.java
View file

@ -158,7 +158,6 @@ public void start(String... args) throws Throwable {
}
};
}
if (UpdateRequest.getController() == null) UpdateRequest.setController(new LauncherUpdateController());
Objects.requireNonNull(args, "args");
if (started.getAndSet(true))
throw new IllegalStateException("Launcher has been already started");

31
Launcher/src/main/java/pro/gravit/launcher/client/LauncherUpdateController.java
View file

@ -1,7 +1,6 @@
package pro.gravit.launcher.client;
import pro.gravit.launcher.NewLauncherSettings;
import pro.gravit.launcher.downloader.ListDownloader;
import pro.gravit.launcher.events.request.UpdateRequestEvent;
import pro.gravit.launcher.hasher.HashedDir;
import pro.gravit.launcher.hasher.HashedEntry;
@ -17,21 +16,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@Deprecated
public class LauncherUpdateController {
public class LauncherUpdateController implements UpdateRequest.UpdateController {
@Override
public void preUpdate(UpdateRequest request, UpdateRequestEvent e) {
}
@Override
public void preDiff(UpdateRequest request, UpdateRequestEvent e) {
}
@Override
public void postDiff(UpdateRequest request, UpdateRequestEvent e, HashedDir.Diff diff) throws IOException {
if (e.zip && e.fullDownload) return;
if (SettingsManager.settings.featureStore) {
@ -116,19 +104,4 @@ public Path tryFind(NewLauncherSettings.HashedStoreEntry en, HashedFile file) th
});
return ret.get();
}
@Override
public void preDownload(UpdateRequest request, UpdateRequestEvent e, List<ListDownloader.DownloadTask> adds) {
}
@Override
public void postDownload(UpdateRequest request, UpdateRequestEvent e) {
}
@Override
public void postUpdate(UpdateRequest request, UpdateRequestEvent e) {
}
}

252
LauncherAPI/src/main/java/pro/gravit/launcher/downloader/ListDownloader.java
View file

@ -1,252 +0,0 @@
package pro.gravit.launcher.downloader;
import org.apache.http.HttpResponse;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import pro.gravit.utils.helper.CommonHelper;
import pro.gravit.utils.helper.IOHelper;
import pro.gravit.utils.helper.LogHelper;
import pro.gravit.utils.helper.VerifyHelper;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public class ListDownloader {
private static final AtomicInteger COUNTER_THR = new AtomicInteger(0);
private static final ThreadFactory FACTORY = r -> CommonHelper.newThread("Downloader Thread #" + COUNTER_THR.incrementAndGet(), true, r);
private static ExecutorService newExecutor() {
return new ThreadPoolExecutor(0, VerifyHelper.verifyInt(Integer.parseInt(System.getProperty("launcher.downloadThreads", "3")), VerifyHelper.POSITIVE, "Thread max count must be positive."), 5, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), FACTORY);
}
@FunctionalInterface
public interface DownloadCallback {
void stateChanged(String filename, long downloadedSize, long size);
}
@FunctionalInterface
public interface DownloadTotalCallback {
void addTotal(long size);
}
public static class DownloadTask {
public final String apply;
public long size;
public final String urlApply;
public DownloadTask(String apply, long size) {
this.apply = apply;
urlApply = apply;
this.size = size;
}
public DownloadTask(String urlApply, String apply, long size) {
this.apply = apply;
this.urlApply = urlApply;
this.size = size;
}
}
public void download(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback) throws IOException, URISyntaxException {
try (CloseableHttpClient httpclient = HttpClients.custom().setUserAgent(IOHelper.USER_AGENT)
.setRedirectStrategy(new LaxRedirectStrategy())
.build()) {
applies.sort(Comparator.comparingLong(a -> a.size));
List<Callable<Void>> toExec = new ArrayList<>();
URI baseUri = new URI(base);
String scheme = baseUri.getScheme();
String host = baseUri.getHost();
int port = baseUri.getPort();
if (port != -1)
host = host + ":" + port;
String path = baseUri.getPath();
List<IOException> excs = new CopyOnWriteArrayList<>();
for (DownloadTask apply : applies) {
URI u = new URI(scheme, host, path + apply.urlApply, "", "");
callback.stateChanged(apply.apply, 0L, apply.size);
Path targetPath = dstDirFile.resolve(apply.apply);
toExec.add(() -> {
if (LogHelper.isDebugEnabled())
LogHelper.debug("Download URL: %s to file %s dir: %s", u.toString(), targetPath.toAbsolutePath().toString(), dstDirFile.toAbsolutePath().toString());
try {
httpclient.execute(new HttpGet(u), new FileDownloadResponseHandler(targetPath, apply, callback, totalCallback, false));
} catch (IOException e) {
excs.add(e);
}
return null;
});
}
try {
ExecutorService e = newExecutor();
e.invokeAll(toExec);
e.shutdown();
e.awaitTermination(4, TimeUnit.HOURS);
} catch (InterruptedException t) {
LogHelper.error(t);
}
if (!excs.isEmpty()) {
IOException toThrow = excs.remove(0);
excs.forEach(toThrow::addSuppressed);
throw toThrow;
}
}
}
public void downloadZip(String base, List<DownloadTask> applies, Path dstDirFile, DownloadCallback callback, DownloadTotalCallback totalCallback, boolean fullDownload) throws IOException {
/*try (CloseableHttpClient httpclient = HttpClients.custom()
.setRedirectStrategy(new LaxRedirectStrategy())
.build()) {
HttpGet get;
URI u = new URL(base).toURI();
LogHelper.debug("Download ZIP URL: %s", u.toString());
get = new HttpGet(u);
httpclient.execute(get, new FileDownloadResponseHandler(dstDirFile, callback, totalCallback, true));
}*/
try (ZipInputStream input = IOHelper.newZipInput(new URL(base))) {
for (ZipEntry entry = input.getNextEntry(); entry != null; entry = input.getNextEntry()) {
if (entry.isDirectory())
continue; // Skip directories
// Unpack entry
String name = entry.getName();
callback.stateChanged(name, 0L, entry.getSize());
LogHelper.subInfo("Downloading file: '%s'", name);
if (fullDownload || applies.stream().anyMatch((t) -> t.apply.equals(name))) {
Path fileName = IOHelper.toPath(name);
transfer(input, dstDirFile.resolve(fileName), fileName.toString(), entry.getSize(), callback, totalCallback);
}
}
}
}
public void downloadOne(String url, Path target) throws IOException, URISyntaxException {
try (CloseableHttpClient httpclient = HttpClients.custom()
.setRedirectStrategy(new LaxRedirectStrategy())
.build()) {
HttpGet get;
URI u = new URL(url).toURI();
if (LogHelper.isDebugEnabled()) {
LogHelper.debug("Download URL: %s", u.toString());
}
get = new HttpGet(u);
httpclient.execute(get, new FileDownloadResponseHandler(target.toAbsolutePath()));
}
}
static class FileDownloadResponseHandler implements ResponseHandler<Path> {
private final Path target;
private final DownloadTask task;
private final DownloadCallback callback;
private final DownloadTotalCallback totalCallback;
private final boolean zip;
public FileDownloadResponseHandler(Path target) {
this.target = target;
this.task = null;
this.zip = false;
callback = null;
totalCallback = null;
}
public FileDownloadResponseHandler(Path target, DownloadTask task, DownloadCallback callback, DownloadTotalCallback totalCallback, boolean zip) {
this.target = target;
this.task = task;
this.callback = callback;
this.totalCallback = totalCallback;
this.zip = zip;
}
public FileDownloadResponseHandler(Path target, DownloadCallback callback, DownloadTotalCallback totalCallback, boolean zip) {
this.target = target;
this.task = null;
this.callback = callback;
this.totalCallback = totalCallback;
this.zip = zip;
}
@Override
public Path handleResponse(HttpResponse response) throws IOException {
InputStream source = response.getEntity().getContent();
int returnCode = response.getStatusLine().getStatusCode();
if (returnCode != 200) {
throw new IllegalStateException(String.format("Request download file %s return code %d", target.toString(), returnCode));
}
long contentLength = response.getEntity().getContentLength();
if (task != null && contentLength != task.size) {
if (task.size > 0)
LogHelper.warning("Missing content length: expected %d | found %d", task.size, contentLength);
else task.size = contentLength;
}
if (zip) {
try (ZipInputStream input = IOHelper.newZipInput(source)) {
ZipEntry entry = input.getNextEntry();
while (entry != null) {
if (entry.isDirectory()) {
entry = input.getNextEntry();
continue;
}
long size = entry.getSize();
String filename = entry.getName();
Path target = this.target.resolve(filename);
if (callback != null) {
callback.stateChanged(entry.getName(), 0, entry.getSize());
}
if (LogHelper.isDevEnabled()) {
LogHelper.dev("Resolved filename %s to %s", filename, target.toAbsolutePath().toString());
}
transfer(source, target, filename, size, callback, totalCallback);
entry = input.getNextEntry();
}
}
return null;
}
if (callback != null && task != null) {
callback.stateChanged(task.apply, 0, task.size);
transfer(source, this.target, task.apply, task.size, callback, totalCallback);
} else
IOHelper.transfer(source, this.target);
return this.target;
}
}
public static void transfer(InputStream input, Path file, String filename, long size, DownloadCallback callback, DownloadTotalCallback totalCallback) throws IOException {
try (OutputStream fileOutput = IOHelper.newOutput(file)) {
long downloaded = 0L;
// Download with digest update
byte[] bytes = IOHelper.newBuffer();
while (downloaded < size) {
int remaining = (int) Math.min(size - downloaded, bytes.length);
int length = input.read(bytes, 0, remaining);
if (length < 0)
throw new EOFException(String.format("%d bytes remaining", size - downloaded));
// Update file
fileOutput.write(bytes, 0, length);
// Update state
downloaded += length;
//totalDownloaded += length;
totalCallback.addTotal(length);
callback.stateChanged(filename, downloaded, size);
}
}
}
}

11
LauncherAPI/src/main/java/pro/gravit/launcher/request/update/LauncherRequest.java
View file

@ -3,7 +3,6 @@
import pro.gravit.launcher.Launcher;
import pro.gravit.launcher.LauncherAPI;
import pro.gravit.launcher.LauncherNetworkAPI;
import pro.gravit.launcher.downloader.ListDownloader;
import pro.gravit.launcher.events.request.LauncherRequestEvent;
import pro.gravit.launcher.request.Request;
import pro.gravit.launcher.request.websockets.StandartClientWebSocketService;
@ -15,6 +14,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@ -60,9 +61,13 @@ public static void update(LauncherRequestEvent result) throws IOException {
IOHelper.transfer(BINARY_PATH, stream);
}*/
try {
ListDownloader downloader = new ListDownloader();
Files.deleteIfExists(C_BINARY_PATH);
downloader.downloadOne(result.url, C_BINARY_PATH);
URL url = new URL(result.url);
URLConnection connection = url.openConnection();
try(InputStream in = connection.getInputStream())
{
IOHelper.transfer(in, C_BINARY_PATH);
}
try (InputStream in = IOHelper.newInput(C_BINARY_PATH)) {
IOHelper.transfer(in, BINARY_PATH);
}

277
LauncherAPI/src/main/java/pro/gravit/launcher/request/update/UpdateRequest.java
View file

@ -1,313 +1,40 @@
package pro.gravit.launcher.request.update;
import pro.gravit.launcher.Launcher;
import pro.gravit.launcher.LauncherAPI;
import pro.gravit.launcher.LauncherNetworkAPI;
import pro.gravit.launcher.downloader.ListDownloader;
import pro.gravit.launcher.events.request.UpdateRequestEvent;
import pro.gravit.launcher.hasher.FileNameMatcher;
import pro.gravit.launcher.hasher.HashedDir;
import pro.gravit.launcher.hasher.HashedEntry;
import pro.gravit.launcher.hasher.HashedFile;
import pro.gravit.launcher.request.Request;
import pro.gravit.launcher.request.update.UpdateRequest.State.Callback;
import pro.gravit.launcher.request.websockets.StandartClientWebSocketService;
import pro.gravit.launcher.request.websockets.WebSocketRequest;
import pro.gravit.utils.helper.IOHelper;
import pro.gravit.utils.helper.LogHelper;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.util.Objects;
public final class UpdateRequest extends Request<UpdateRequestEvent> implements WebSocketRequest {
public interface UpdateController {
void preUpdate(UpdateRequest request, UpdateRequestEvent e);
void preDiff(UpdateRequest request, UpdateRequestEvent e);
void postDiff(UpdateRequest request, UpdateRequestEvent e, HashedDir.Diff diff) throws IOException;
void preDownload(UpdateRequest request, UpdateRequestEvent e, List<ListDownloader.DownloadTask> adds);
void postDownload(UpdateRequest request, UpdateRequestEvent e);
void postUpdate(UpdateRequest request, UpdateRequestEvent e);
}
private static UpdateController controller;
public static void setController(UpdateController controller) {
UpdateRequest.controller = controller;
}
public static UpdateController getController() {
return controller;
}
@Override
public String getType() {
return "update";
}
public static final class State {
@FunctionalInterface
public interface Callback {
@LauncherAPI
void call(State state);
}
@LauncherAPI
public final long fileDownloaded;
@LauncherAPI
public final long fileSize;
@LauncherAPI
public final long totalDownloaded;
@LauncherAPI
public final long totalSize;
@LauncherAPI
public final String filePath;
@LauncherAPI
public final Duration duration;
public State(String filePath, long fileDownloaded, long fileSize, long totalDownloaded, long totalSize, Duration duration) {
this.filePath = filePath;
this.fileDownloaded = fileDownloaded;
this.fileSize = fileSize;
this.totalDownloaded = totalDownloaded;
this.totalSize = totalSize;
// Also store time of creation
this.duration = duration;
}
@LauncherAPI
public double getBps() {
long seconds = duration.getSeconds();
if (seconds == 0)
return -1.0D; // Otherwise will throw /0 exception
return totalDownloaded / (double) seconds;
}
@LauncherAPI
public Duration getEstimatedTime() {
double bps = getBps();
if (bps <= 0.0D)
return null; // Otherwise will throw /0 exception
return Duration.ofSeconds((long) (getTotalRemaining() / bps));
}
@LauncherAPI
public double getFileDownloadedKiB() {
return fileDownloaded / 1024.0D;
}
@LauncherAPI
public double getFileDownloadedMiB() {
return getFileDownloadedKiB() / 1024.0D;
}
@LauncherAPI
public double getFileDownloadedPart() {
if (fileSize == 0)
return 0.0D;
return (double) fileDownloaded / fileSize;
}
@LauncherAPI
public long getFileRemaining() {
return fileSize - fileDownloaded;
}
@LauncherAPI
public double getFileRemainingKiB() {
return getFileRemaining() / 1024.0D;
}
@LauncherAPI
public double getFileRemainingMiB() {
return getFileRemainingKiB() / 1024.0D;
}
@LauncherAPI
public double getFileSizeKiB() {
return fileSize / 1024.0D;
}
@LauncherAPI
public double getFileSizeMiB() {
return getFileSizeKiB() / 1024.0D;
}
@LauncherAPI
public double getTotalDownloadedKiB() {
return totalDownloaded / 1024.0D;
}
@LauncherAPI
public double getTotalDownloadedMiB() {
return getTotalDownloadedKiB() / 1024.0D;
}
@LauncherAPI
public double getTotalDownloadedPart() {
if (totalSize == 0)
return 0.0D;
return (double) totalDownloaded / totalSize;
}
@LauncherAPI
public long getTotalRemaining() {
return totalSize - totalDownloaded;
}
@LauncherAPI
public double getTotalRemainingKiB() {
return getTotalRemaining() / 1024.0D;
}
@LauncherAPI
public double getTotalRemainingMiB() {
return getTotalRemainingKiB() / 1024.0D;
}
@LauncherAPI
public double getTotalSizeKiB() {
return totalSize / 1024.0D;
}
@LauncherAPI
public double getTotalSizeMiB() {
return getTotalSizeKiB() / 1024.0D;
}
}
@Override
public UpdateRequestEvent requestDo(StandartClientWebSocketService service) throws Exception {
LogHelper.debug("Start update request");
UpdateRequestEvent e = (UpdateRequestEvent) service.sendRequest(this);
if (controller != null) controller.preUpdate(this, e);
LogHelper.debug("Start update");
Launcher.profile.pushOptionalFile(e.hdir, !Launcher.profile.isUpdateFastCheck());
if (controller != null) controller.preDiff(this, e);
HashedDir.Diff diff = e.hdir.diff(localDir, matcher);
if (controller != null) controller.postDiff(this, e, diff);
final List<ListDownloader.DownloadTask> adds = new ArrayList<>();
if (controller != null) controller.preDownload(this, e, adds);
diff.mismatch.walk(IOHelper.CROSS_SEPARATOR, (path, name, entry) -> {
if (entry.getType().equals(HashedEntry.Type.FILE)) {
if (!entry.flag) {
HashedFile file = (HashedFile) entry;
totalSize += file.size;
adds.add(new ListDownloader.DownloadTask(path, file.size));
}
} else if (entry.getType().equals(HashedEntry.Type.DIR)) {
try {
Files.createDirectories(dir.resolve(path));
} catch (IOException ex) {
LogHelper.error(ex);
}
}
return HashedDir.WalkAction.CONTINUE;
});
totalSize = diff.mismatch.size();
startTime = Instant.now();
updateState("UnknownFile", 0L, 100);
ListDownloader listDownloader = new ListDownloader();
LogHelper.info("Download %s to %s", dirName, dir.toAbsolutePath().toString());
if (e.zip && !adds.isEmpty()) {
listDownloader.downloadZip(e.url, adds, dir, this::updateState, (add) -> totalDownloaded += add, e.fullDownload);
} else {
listDownloader.download(e.url, adds, dir, this::updateState, (add) -> totalDownloaded += add);
}
if (controller != null) controller.postDownload(this, e);
deleteExtraDir(dir, diff.extra, diff.extra.flag);
if (controller != null) controller.postUpdate(this, e);
LogHelper.debug("Update success");
return e;
return (UpdateRequestEvent) service.sendRequest(this);
}
// Instance
@LauncherNetworkAPI
private final String dirName;
private transient final Path dir;
public Path getDir() {
return dir;
}
private transient final FileNameMatcher matcher;
private transient final boolean digest;
private transient volatile Callback stateCallback;
// State
private transient HashedDir localDir;
private transient long totalDownloaded;
private transient long totalSize;
private transient Instant startTime;
@LauncherAPI
public UpdateRequest(String dirName, Path dir, FileNameMatcher matcher, boolean digest) {
this.dirName = IOHelper.verifyFileName(dirName);
public UpdateRequest(Path dir) {
this.dir = Objects.requireNonNull(dir, "dir");
this.matcher = matcher;
this.digest = digest;
}
private void deleteExtraDir(Path subDir, HashedDir subHDir, boolean flag) throws IOException {
for (Entry<String, HashedEntry> mapEntry : subHDir.map().entrySet()) {
String name = mapEntry.getKey();
Path path = subDir.resolve(name);
// Delete list and dirs based on type
HashedEntry entry = mapEntry.getValue();
HashedEntry.Type entryType = entry.getType();
switch (entryType) {
case FILE:
updateState(IOHelper.toString(path), 0, 0);
Files.delete(path);
break;
case DIR:
deleteExtraDir(path, (HashedDir) entry, flag || entry.flag);
break;
default:
throw new AssertionError("Unsupported hashed entry type: " + entryType.name());
}
}
// Delete!
if (flag) {
updateState(IOHelper.toString(subDir), 0, 0);
Files.delete(subDir);
}
}
@Override
public UpdateRequestEvent request() throws Exception {
Files.createDirectories(dir);
localDir = new HashedDir(dir, matcher, false, digest);
// Start request
return super.request();
}
@LauncherAPI
public void setStateCallback(Callback callback) {
stateCallback = callback;
}
private void updateState(String filePath, long fileDownloaded, long fileSize) {
if (stateCallback != null)
stateCallback.call(new State(filePath, fileDownloaded, fileSize,
totalDownloaded, totalSize, Duration.between(startTime, Instant.now())));
}
}

2
modules

@ -1 +1 @@
Subproject commit 3cf0cb41a64b1f05d76bdc67f83d67d80ebe41f1
Subproject commit 5069ebacddc00b261dd03036fc637a8aa803e2d5