[FEATURE] Cкачивание zip архивами - часть 2 обработка со стороны лаунчсервера.

LaunchServer/src/main/java/pro/gravit/launchserver/LaunchServer.java

@@ -33,6 +33,7 @@
 import java.util.Timer;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.zip.CRC32;
+import java.util.zip.ZipOutputStream;
 
 import io.netty.handler.logging.LogLevel;
 import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
@@ -43,6 +44,7 @@
 import pro.gravit.launcher.LauncherConfig;
 import pro.gravit.launcher.NeedGarbageCollection;
 import pro.gravit.launcher.hasher.HashedDir;
+import pro.gravit.launcher.hasher.HashedEntry;
 import pro.gravit.launcher.hwid.HWIDProvider;
 import pro.gravit.launcher.managers.ConfigManager;
 import pro.gravit.launcher.managers.GarbageManager;
@@ -67,6 +69,7 @@
 import pro.gravit.launchserver.components.RegLimiterComponent;
 import pro.gravit.launchserver.config.LaunchServerRuntimeConfig;
 import pro.gravit.launchserver.dao.provider.DaoProvider;
+import pro.gravit.launchserver.hasher.UpdateData;
 import pro.gravit.launchserver.manangers.*;
 import pro.gravit.launchserver.manangers.hook.AuthHookManager;
 import pro.gravit.launchserver.manangers.hook.BuildHookManager;
@@ -282,6 +285,7 @@ public class NettyConfig {
         public boolean fileServerEnabled;
         public boolean sendExceptionEnabled;
         public boolean ipForwarding;
+        public boolean directoryListing = true;
         public String launcherURL;
         public String downloadURL;
         public String launcherEXEURL;
@@ -463,6 +467,8 @@ public static void main(String... args) throws Throwable {
 
     public static Class<? extends LauncherBinary> defaultLauncherEXEBinaryClass = null;
 
+	public final Path optimizedUpdatesDir;
+    
     public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException, InvalidKeySpecException {
         this.dir = dir;
         this.testEnv = testEnv;
@@ -489,6 +495,7 @@ public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException
         publicKeyFile = dir.resolve("public.key");
         privateKeyFile = dir.resolve("private.key");
         updatesDir = dir.resolve("updates");
+        optimizedUpdatesDir = dir.resolve("optimized_updates");
         profilesDir = dir.resolve("profiles");
 
         caCertFile = dir.resolve("ca.crt");
@@ -681,6 +688,9 @@ public LaunchServer(Path dir, boolean testEnv, String[] args) throws IOException
         // Sync updates dir
         if (!IOHelper.isDir(updatesDir))
             Files.createDirectory(updatesDir);
+        if (!IOHelper.isDir(optimizedUpdatesDir))
+            Files.createDirectory(optimizedUpdatesDir);
+        updatesDirMap = null; // small hack
         syncUpdatesDir(null);
 
         // Sync profiles dir
@@ -818,6 +828,7 @@ private void generateConfigIfNotExists(boolean testEnv) throws IOException {
         newConfig.stripLineNumbers = true;
         newConfig.deleteTempFiles = true;
         newConfig.isWarningMissArchJava = true;
+        newConfig.zipDownload = true;
 
         newConfig.components = new HashMap<>();
         AuthLimiterComponent authLimiterComponent = new AuthLimiterComponent();
@@ -931,6 +942,7 @@ public void syncProfilesDir() throws IOException {
     public void syncUpdatesDir(Collection<String> dirs) throws IOException {
         LogHelper.info("Syncing updates dir");
         Map<String, HashedDir> newUpdatesDirMap = new HashMap<>(16);
+        boolean work = updatesDirMap != null;
         try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(updatesDir)) {
             for (final Path updateDir : dirStream) {
                 if (Files.isHidden(updateDir))
@@ -956,12 +968,39 @@ public void syncUpdatesDir(Collection<String> dirs) throws IOException {
                 // Sync and sign update dir
                 LogHelper.info("Syncing '%s' update dir", name);
                 HashedDir updateHDir = new HashedDir(updateDir, null, true, true);
+                if (work && config.zipDownload) processUpdate(updateDir, updateHDir, name);
                 newUpdatesDirMap.put(name, updateHDir);
             }
         }
         updatesDirMap = Collections.unmodifiableMap(newUpdatesDirMap);
     }
 
+    private void processUpdate(Path updateDir, HashedDir updateHDir, String name) throws IOException {
+    	updateHDir.walk(IOHelper.CROSS_SEPARATOR, (path, filename, entry) -> {
+    		if (entry.getType().equals(HashedEntry.Type.DIR)) {
+                if (UpdateData.needsZip((HashedDir) entry)) {
+                	Path p = updateDir.resolve(path);
+            		Path out = optimizedUpdatesDir.resolve(name).resolve(path + ".zip");
+                	try (ZipOutputStream compressed = new ZipOutputStream(IOHelper.newOutput(out))) {
+                		IOHelper.walk(p, new SimpleFileVisitor<Path>() {
+                			@Override
+                			public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                				compressed.putNextEntry(IOHelper.newZipEntry(
+                						p.relativize(file).toString()
+                						.replace(IOHelper.PLATFORM_SEPARATOR, IOHelper.CROSS_SEPARATOR)));
+                				IOHelper.transfer(file, compressed);
+                				return super.visitFile(file, attrs);
+                			}
+                		}, true);
+                	}
+                    return HashedDir.WalkAction.SKIP_DIR;
+                }
+            }
+    		return HashedDir.WalkAction.CONTINUE;
+    	});
+
+	}
+    
     public void restart() {
         ProcessBuilder builder = new ProcessBuilder();
         if (config.startScript != null) builder.command(Collections.singletonList(config.startScript));

LaunchServer/src/main/java/pro/gravit/launchserver/hasher/UpdateData.java

@@ -0,0 +1,47 @@
+package pro.gravit.launchserver.hasher;
+
+import java.io.IOError;
+import java.io.IOException;
+import pro.gravit.launcher.hasher.HashedDir;
+import pro.gravit.launcher.hasher.HashedEntry;
+import pro.gravit.utils.helper.IOHelper;
+
+public final class UpdateData {
+	private static final class Pair {
+		private int cnt = 0;
+		private long size = 0;
+	}
+	public static final int COUNT_LIMIT = 4;
+	public static final long AVG_LIMIT = IOHelper.MB*4;
+	public static final long TOTAL_LIMIT = IOHelper.MB*16;
+	public static final double SIMPLE_DOWNLOAD_SIZE_COFF = 0.75D;
+	public static final double SIMPLE_DOWNLOAD_FILE_COPF = 0.5D;
+
+	private UpdateData() {
+	}
+
+	public static boolean needsZip(Pair p) {
+        long avg = p.size/(long)p.cnt;
+        return p.size < TOTAL_LIMIT && avg < AVG_LIMIT && p.cnt > COUNT_LIMIT;
+	}
+
+	public static boolean needsZip(HashedDir hDir) {
+        return needsZip(count(hDir));
+	}
+
+	private static Pair count(HashedDir hDir) {
+		final Pair pair = new Pair();
+        try {
+			hDir.walk(IOHelper.CROSS_SEPARATOR, (p,n,e) -> {
+				if (e.getType().equals(HashedEntry.Type.FILE)) {
+					pair.cnt++;
+					pair.size += e.size();
+				}
+				return HashedDir.WalkAction.CONTINUE;
+			});
+		} catch (IOException e) {
+			throw new IOError(e); // never happen
+		}
+        return pair;
+	}
+}

LaunchServer/src/main/java/pro/gravit/launchserver/socket/LauncherNettyServer.java

@@ -1,6 +1,7 @@
 package pro.gravit.launchserver.socket;
 
 import java.net.InetSocketAddress;
+import java.util.Arrays;
 
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.channel.ChannelFuture;
@@ -63,7 +64,7 @@ public void initChannel(SocketChannel ch) {
                         pipeline.addLast(new WebSocketServerCompressionHandler());
                         pipeline.addLast(new WebSocketServerProtocolHandler(WEBSOCKET_PATH, null, true));
                         if (server.config.netty.fileServerEnabled)
-                            pipeline.addLast(new FileServerHandler(server.updatesDir, true));
+                            pipeline.addLast(new FileServerHandler(Arrays.asList(server.updatesDir, server.optimizedUpdatesDir), server.config.netty.directoryListing));
                         pipeline.addLast(new WebSocketFrameHandler(context, server, service));
                     }
                 });

LaunchServer/src/main/java/pro/gravit/launchserver/socket/handlers/fileserver/FileServerHandler.java

@@ -22,7 +22,9 @@
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
+import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.TimeZone;
 import java.util.regex.Pattern;
 
@@ -48,6 +50,7 @@
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.handler.stream.ChunkedFile;
 import io.netty.util.CharsetUtil;
+import pro.gravit.utils.helper.IOHelper;
 
 public class FileServerHandler extends SimpleChannelInboundHandler<FullHttpRequest> {
 
@@ -56,10 +59,10 @@ public class FileServerHandler extends SimpleChannelInboundHandler<FullHttpReque
     public static final String READ = "r";
     public static final int HTTP_CACHE_SECONDS = 60;
 	private static final boolean OLD_ALGO = Boolean.parseBoolean(System.getProperty("launcher.fileserver.oldalgo", "true"));
-    private final Path base;
+    private final List<Path> base;
     private final boolean fullOut;
 
-    public FileServerHandler(Path base, boolean fullOut) {
+    public FileServerHandler(List<Path> base, boolean fullOut) {
         this.base = base;
         this.fullOut = fullOut;
     }
@@ -83,12 +86,12 @@ public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) thr
             return;
         }
 
-        File file = base.resolve(path).toFile();
+        Optional<File> fileO = base.stream().map(t -> t.resolve(path)).filter(t -> IOHelper.exists(t) && !IOHelper.isHidden(t)).map(t -> t.toFile()).findFirst();
-        if (file.isHidden() || !file.exists()) {
+        if (!fileO.isPresent()) {
             sendError(ctx, NOT_FOUND);
             return;
         }
-
+        File file = fileO.get();
         if (file.isDirectory()) {
             if (fullOut) {
                 if (uri.endsWith("/")) {
@@ -172,8 +175,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
         }
     }
 
-    private static final Pattern INSECURE_URI = Pattern.compile(".*[<>&\"].*");
-
     private static String sanitizeUri(String uri) {
         // Decode the path.
         try {

Launcher/runtime/dialog/dialog.js

@@ -203,10 +203,6 @@ function goAuth(event) {
     }
 
      var rsaPassword = null;
-     var auth = authOptions.getSelectionModel().getSelectedItem();
-     if (auth === null) {
-        return;
-     }
      if (!passwordField.isDisable()) {
          var password = passwordField.getText();
          if (password !== null && !password.isEmpty()) {

LauncherCore/src/main/java/pro/gravit/utils/helper/IOHelper.java

@@ -168,6 +168,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce
     private static final Pattern CROSS_SEPARATOR_PATTERN = Pattern.compile(CROSS_SEPARATOR, Pattern.LITERAL);
 
     private static final Pattern PLATFORM_SEPARATOR_PATTERN = Pattern.compile(PLATFORM_SEPARATOR, Pattern.LITERAL);
+	public static final int MB = 1 << 20;
 
     @LauncherAPI
     public static void close(AutoCloseable closeable) {
@@ -767,6 +768,15 @@ public static void write(Path file, byte[] bytes) throws IOException {
         Files.write(file, bytes, WRITE_OPTIONS);
     }
 
+    @LauncherAPI
+    public static boolean isHidden(Path path) {
+        try {
+			return Files.isHidden(path);
+		} catch (IOException e) {
+			return false;
+		}
+    }
+
     private IOHelper() {
     }
 }