From 4ab67460e29751032f9df62294b51d7e544e3be1 Mon Sep 17 00:00:00 2001 From: Zaxar163 Date: Fri, 19 Jul 2019 18:00:19 +0300 Subject: [PATCH] =?UTF-8?q?[FEATURE]=20C=D0=BA=D0=B0=D1=87=D0=B8=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B5=20zip=20=D0=B0=D1=80=D1=85=D0=B8=D0=B2?= =?UTF-8?q?=D0=B0=D0=BC=D0=B8=20-=20=D1=87=D0=B0=D1=81=D1=82=D1=8C=202=20?= =?UTF-8?q?=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B0=20=D1=81?= =?UTF-8?q?=D0=BE=20=D1=81=D1=82=D0=BE=D1=80=D0=BE=D0=BD=D1=8B=20=D0=BB?= =?UTF-8?q?=D0=B0=D1=83=D0=BD=D1=87=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pro/gravit/launchserver/LaunchServer.java | 39 +++++++++++++++ .../launchserver/hasher/UpdateData.java | 47 +++++++++++++++++++ .../socket/LauncherNettyServer.java | 3 +- .../fileserver/FileServerHandler.java | 15 +++--- Launcher/runtime/dialog/dialog.js | 4 -- .../pro/gravit/utils/helper/IOHelper.java | 10 ++++ 6 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 LaunchServer/src/main/java/pro/gravit/launchserver/hasher/UpdateData.java diff --git a/LaunchServer/src/main/java/pro/gravit/launchserver/LaunchServer.java b/LaunchServer/src/main/java/pro/gravit/launchserver/LaunchServer.java index d46edfd5..1ddbe627 100644 --- a/LaunchServer/src/main/java/pro/gravit/launchserver/LaunchServer.java +++ b/LaunchServer/src/main/java/pro/gravit/launchserver/LaunchServer.java @@ -33,6 +33,7 @@ import java.util.Timer; import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.CRC32; +import java.util.zip.ZipOutputStream; import io.netty.handler.logging.LogLevel; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; @@ -43,6 +44,7 @@ import pro.gravit.launcher.LauncherConfig; import pro.gravit.launcher.NeedGarbageCollection; import pro.gravit.launcher.hasher.HashedDir; +import pro.gravit.launcher.hasher.HashedEntry; import pro.gravit.launcher.hwid.HWIDProvider; import pro.gravit.launcher.managers.ConfigManager; import pro.gravit.launcher.managers.GarbageManager; @@ -67,6 +69,7 @@ import pro.gravit.launchserver.components.RegLimiterComponent; import pro.gravit.launchserver.config.LaunchServerRuntimeConfig; import pro.gravit.launchserver.dao.provider.DaoProvider; +import pro.gravit.launchserver.hasher.UpdateData; import pro.gravit.launchserver.manangers.*; import pro.gravit.launchserver.manangers.hook.AuthHookManager; import pro.gravit.launchserver.manangers.hook.BuildHookManager; @@ -282,6 +285,7 @@ public class NettyConfig { public boolean fileServerEnabled; public boolean sendExceptionEnabled; public boolean ipForwarding; + public boolean directoryListing = true; public String launcherURL; public String downloadURL; public String launcherEXEURL; @@ -463,6 +467,8 @@ public static void main(String... args) throws Throwable { public static Class defaultLauncherEXEBinaryClass = null; + public final Path optimizedUpdatesDir; + public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException, InvalidKeySpecException { this.dir = dir; this.testEnv = testEnv; @@ -489,6 +495,7 @@ public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException publicKeyFile = dir.resolve("public.key"); privateKeyFile = dir.resolve("private.key"); updatesDir = dir.resolve("updates"); + optimizedUpdatesDir = dir.resolve("optimized_updates"); profilesDir = dir.resolve("profiles"); caCertFile = dir.resolve("ca.crt"); @@ -681,6 +688,9 @@ public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException // Sync updates dir if (!IOHelper.isDir(updatesDir)) Files.createDirectory(updatesDir); + if (!IOHelper.isDir(optimizedUpdatesDir)) + Files.createDirectory(optimizedUpdatesDir); + updatesDirMap = null; // small hack syncUpdatesDir(null); // Sync profiles dir @@ -818,6 +828,7 @@ private void generateConfigIfNotExists(boolean testEnv) throws IOException { newConfig.stripLineNumbers = true; newConfig.deleteTempFiles = true; newConfig.isWarningMissArchJava = true; + newConfig.zipDownload = true; newConfig.components = new HashMap<>(); AuthLimiterComponent authLimiterComponent = new AuthLimiterComponent(); @@ -931,6 +942,7 @@ public void syncProfilesDir() throws IOException { public void syncUpdatesDir(Collection dirs) throws IOException { LogHelper.info("Syncing updates dir"); Map newUpdatesDirMap = new HashMap<>(16); + boolean work = updatesDirMap != null; try (DirectoryStream dirStream = Files.newDirectoryStream(updatesDir)) { for (final Path updateDir : dirStream) { if (Files.isHidden(updateDir)) @@ -956,12 +968,39 @@ public void syncUpdatesDir(Collection dirs) throws IOException { // Sync and sign update dir LogHelper.info("Syncing '%s' update dir", name); HashedDir updateHDir = new HashedDir(updateDir, null, true, true); + if (work && config.zipDownload) processUpdate(updateDir, updateHDir, name); newUpdatesDirMap.put(name, updateHDir); } } updatesDirMap = Collections.unmodifiableMap(newUpdatesDirMap); } + private void processUpdate(Path updateDir, HashedDir updateHDir, String name) throws IOException { + updateHDir.walk(IOHelper.CROSS_SEPARATOR, (path, filename, entry) -> { + if (entry.getType().equals(HashedEntry.Type.DIR)) { + if (UpdateData.needsZip((HashedDir) entry)) { + Path p = updateDir.resolve(path); + Path out = optimizedUpdatesDir.resolve(name).resolve(path + ".zip"); + try (ZipOutputStream compressed = new ZipOutputStream(IOHelper.newOutput(out))) { + IOHelper.walk(p, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + compressed.putNextEntry(IOHelper.newZipEntry( + p.relativize(file).toString() + .replace(IOHelper.PLATFORM_SEPARATOR, IOHelper.CROSS_SEPARATOR))); + IOHelper.transfer(file, compressed); + return super.visitFile(file, attrs); + } + }, true); + } + return HashedDir.WalkAction.SKIP_DIR; + } + } + return HashedDir.WalkAction.CONTINUE; + }); + + } + public void restart() { ProcessBuilder builder = new ProcessBuilder(); if (config.startScript != null) builder.command(Collections.singletonList(config.startScript)); diff --git a/LaunchServer/src/main/java/pro/gravit/launchserver/hasher/UpdateData.java b/LaunchServer/src/main/java/pro/gravit/launchserver/hasher/UpdateData.java new file mode 100644 index 00000000..a518108b --- /dev/null +++ b/LaunchServer/src/main/java/pro/gravit/launchserver/hasher/UpdateData.java @@ -0,0 +1,47 @@ +package pro.gravit.launchserver.hasher; + +import java.io.IOError; +import java.io.IOException; +import pro.gravit.launcher.hasher.HashedDir; +import pro.gravit.launcher.hasher.HashedEntry; +import pro.gravit.utils.helper.IOHelper; + +public final class UpdateData { + private static final class Pair { + private int cnt = 0; + private long size = 0; + } + public static final int COUNT_LIMIT = 4; + public static final long AVG_LIMIT = IOHelper.MB*4; + public static final long TOTAL_LIMIT = IOHelper.MB*16; + public static final double SIMPLE_DOWNLOAD_SIZE_COFF = 0.75D; + public static final double SIMPLE_DOWNLOAD_FILE_COPF = 0.5D; + + private UpdateData() { + } + + public static boolean needsZip(Pair p) { + long avg = p.size/(long)p.cnt; + return p.size < TOTAL_LIMIT && avg < AVG_LIMIT && p.cnt > COUNT_LIMIT; + } + + public static boolean needsZip(HashedDir hDir) { + return needsZip(count(hDir)); + } + + private static Pair count(HashedDir hDir) { + final Pair pair = new Pair(); + try { + hDir.walk(IOHelper.CROSS_SEPARATOR, (p,n,e) -> { + if (e.getType().equals(HashedEntry.Type.FILE)) { + pair.cnt++; + pair.size += e.size(); + } + return HashedDir.WalkAction.CONTINUE; + }); + } catch (IOException e) { + throw new IOError(e); // never happen + } + return pair; + } +} \ No newline at end of file diff --git a/LaunchServer/src/main/java/pro/gravit/launchserver/socket/LauncherNettyServer.java b/LaunchServer/src/main/java/pro/gravit/launchserver/socket/LauncherNettyServer.java index c0b5b205..8149e8b8 100644 --- a/LaunchServer/src/main/java/pro/gravit/launchserver/socket/LauncherNettyServer.java +++ b/LaunchServer/src/main/java/pro/gravit/launchserver/socket/LauncherNettyServer.java @@ -1,6 +1,7 @@ package pro.gravit.launchserver.socket; import java.net.InetSocketAddress; +import java.util.Arrays; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; @@ -63,7 +64,7 @@ public void initChannel(SocketChannel ch) { pipeline.addLast(new WebSocketServerCompressionHandler()); pipeline.addLast(new WebSocketServerProtocolHandler(WEBSOCKET_PATH, null, true)); if (server.config.netty.fileServerEnabled) - pipeline.addLast(new FileServerHandler(server.updatesDir, true)); + pipeline.addLast(new FileServerHandler(Arrays.asList(server.updatesDir, server.optimizedUpdatesDir), server.config.netty.directoryListing)); pipeline.addLast(new WebSocketFrameHandler(context, server, service)); } }); diff --git a/LaunchServer/src/main/java/pro/gravit/launchserver/socket/handlers/fileserver/FileServerHandler.java b/LaunchServer/src/main/java/pro/gravit/launchserver/socket/handlers/fileserver/FileServerHandler.java index 88e762ac..b330af6e 100644 --- a/LaunchServer/src/main/java/pro/gravit/launchserver/socket/handlers/fileserver/FileServerHandler.java +++ b/LaunchServer/src/main/java/pro/gravit/launchserver/socket/handlers/fileserver/FileServerHandler.java @@ -22,7 +22,9 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; +import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.TimeZone; import java.util.regex.Pattern; @@ -48,6 +50,7 @@ import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.stream.ChunkedFile; import io.netty.util.CharsetUtil; +import pro.gravit.utils.helper.IOHelper; public class FileServerHandler extends SimpleChannelInboundHandler { @@ -56,10 +59,10 @@ public class FileServerHandler extends SimpleChannelInboundHandler base; private final boolean fullOut; - public FileServerHandler(Path base, boolean fullOut) { + public FileServerHandler(List base, boolean fullOut) { this.base = base; this.fullOut = fullOut; } @@ -83,12 +86,12 @@ public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) thr return; } - File file = base.resolve(path).toFile(); - if (file.isHidden() || !file.exists()) { + Optional fileO = base.stream().map(t -> t.resolve(path)).filter(t -> IOHelper.exists(t) && !IOHelper.isHidden(t)).map(t -> t.toFile()).findFirst(); + if (!fileO.isPresent()) { sendError(ctx, NOT_FOUND); return; } - + File file = fileO.get(); if (file.isDirectory()) { if (fullOut) { if (uri.endsWith("/")) { @@ -172,8 +175,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { } } - private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*"); - private static String sanitizeUri(String uri) { // Decode the path. try { diff --git a/Launcher/runtime/dialog/dialog.js b/Launcher/runtime/dialog/dialog.js index 1c23beb4..62a0400f 100644 --- a/Launcher/runtime/dialog/dialog.js +++ b/Launcher/runtime/dialog/dialog.js @@ -203,10 +203,6 @@ function goAuth(event) { } var rsaPassword = null; - var auth = authOptions.getSelectionModel().getSelectedItem(); - if (auth === null) { - return; - } if (!passwordField.isDisable()) { var password = passwordField.getText(); if (password !== null && !password.isEmpty()) { diff --git a/LauncherCore/src/main/java/pro/gravit/utils/helper/IOHelper.java b/LauncherCore/src/main/java/pro/gravit/utils/helper/IOHelper.java index 77b34f1b..4312d54c 100644 --- a/LauncherCore/src/main/java/pro/gravit/utils/helper/IOHelper.java +++ b/LauncherCore/src/main/java/pro/gravit/utils/helper/IOHelper.java @@ -168,6 +168,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce private static final Pattern CROSS_SEPARATOR_PATTERN = Pattern.compile(CROSS_SEPARATOR, Pattern.LITERAL); private static final Pattern PLATFORM_SEPARATOR_PATTERN = Pattern.compile(PLATFORM_SEPARATOR, Pattern.LITERAL); + public static final int MB = 1 << 20; @LauncherAPI public static void close(AutoCloseable closeable) { @@ -767,6 +768,15 @@ public static void write(Path file, byte[] bytes) throws IOException { Files.write(file, bytes, WRITE_OPTIONS); } + @LauncherAPI + public static boolean isHidden(Path path) { + try { + return Files.isHidden(path); + } catch (IOException e) { + return false; + } + } + private IOHelper() { } }