Some improvements in FileDownloader (control of request params).

This commit is contained in:
zaxar163 2018-10-01 19:46:20 +03:00
parent fccd793168
commit bc6313b546

View file

@ -7,6 +7,9 @@
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.security.cert.Certificate; import java.security.cert.Certificate;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
@ -22,10 +25,14 @@ public static interface Handler {
public void check(Certificate[] certs) throws IOException; public void check(Certificate[] certs) throws IOException;
} }
public static final Map<String, String> requestClient = Collections.singletonMap("User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
public static final int INTERVAL = 300; public static final int INTERVAL = 300;
private final File file; private final File file;
private final URL url; private final URL url;
private final String method;
public final Map<String, String> requestProps;
public AtomicInteger writed = new AtomicInteger(0); public AtomicInteger writed = new AtomicInteger(0);
public final AtomicBoolean interrupt = new AtomicBoolean(false); public final AtomicBoolean interrupt = new AtomicBoolean(false);
public final AtomicBoolean interrupted = new AtomicBoolean(false); public final AtomicBoolean interrupted = new AtomicBoolean(false);
@ -34,37 +41,84 @@ public static interface Handler {
private final Handler handler; private final Handler handler;
public Downloader(URL url, File file) { public Downloader(URL url, File file) {
this.requestProps = new HashMap<>(requestClient);
this.file = file; this.file = file;
this.url = url; this.url = url;
this.skip = 0; this.skip = 0;
this.handler = null; this.handler = null;
this.method = null;
} }
public Downloader(URL url, File file, int skip) { public Downloader(URL url, File file, int skip) {
this.requestProps = new HashMap<>(requestClient);
this.file = file; this.file = file;
this.url = url; this.url = url;
this.skip = skip; this.skip = skip;
this.handler = null; this.handler = null;
this.method = null;
} }
public Downloader(URL url, File file, Handler handler) { public Downloader(URL url, File file, Handler handler) {
this.requestProps = new HashMap<>(requestClient);
this.file = file; this.file = file;
this.url = url; this.url = url;
this.skip = 0; this.skip = 0;
this.handler = handler; this.handler = handler;
this.method = null;
} }
public Downloader(URL url, File file, int skip, Handler handler) { public Downloader(URL url, File file, int skip, Handler handler) {
this.requestProps = new HashMap<>(requestClient);
this.file = file; this.file = file;
this.url = url; this.url = url;
this.skip = skip; this.skip = skip;
this.handler = handler; this.handler = handler;
this.method = null;
}
public Downloader(URL url, File file, int skip, Handler handler, Map<String, String> requestProps) {
this.requestProps = new HashMap<>(requestProps);
this.file = file;
this.url = url;
this.skip = skip;
this.handler = handler;
this.method = null;
}
public Downloader(URL url, File file, int skip, Handler handler, Map<String, String> requestProps, String method) {
this.requestProps = new HashMap<>(requestProps);
this.file = file;
this.url = url;
this.skip = skip;
this.handler = handler;
this.method = method;
}
public Downloader(URL url, File file, int skip, Handler handler, String method) {
this.requestProps = new HashMap<>(requestClient);
this.file = file;
this.url = url;
this.skip = skip;
this.handler = handler;
this.method = method;
}
public Map<String, String> getProps() {
return requestProps;
}
public void addProp(String key, String value) {
requestProps.put(key, value);
} }
public File getFile() { public File getFile() {
return file; return file;
} }
public String getMethod() {
return method;
}
public Handler getHandler() { public Handler getHandler() {
return handler; return handler;
} }
@ -75,16 +129,16 @@ public void downloadFile() throws IOException {
interrupted.set(false); interrupted.set(false);
if (url.getProtocol().equalsIgnoreCase("http")) { if (url.getProtocol().equalsIgnoreCase("http")) {
HttpURLConnection connect = (HttpURLConnection) (url).openConnection(); HttpURLConnection connect = (HttpURLConnection) (url).openConnection();
connect.setRequestProperty("User-Agent", if (method != null) connect.setRequestMethod(method);
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); // for for (Map.Entry<String, String> ent : requestProps.entrySet()) {
// stupid connect.setRequestProperty(ent.getKey(), ent.getValue());
// servers }
connect.setInstanceFollowRedirects(true); connect.setInstanceFollowRedirects(true);
if (!(connect.getResponseCode() >= 200 && connect.getResponseCode() < 300)) if (!(connect.getResponseCode() >= 200 && connect.getResponseCode() < 300))
throw new IOException(String.format("Invalid response of http server %d.", connect.getResponseCode())); throw new IOException(String.format("Invalid response of http server %d.", connect.getResponseCode()));
try (BufferedInputStream in = new BufferedInputStream(connect.getInputStream(), IOHelper.BUFFER_SIZE); try (BufferedInputStream in = new BufferedInputStream(connect.getInputStream(), IOHelper.BUFFER_SIZE);
FileOutputStream fout = new FileOutputStream(file, skip != 0)) { FileOutputStream fout = new FileOutputStream(file, skip != 0)) {
final byte data[] = new byte[IOHelper.BUFFER_SIZE]; byte data[] = new byte[IOHelper.BUFFER_SIZE];
int count = -1; int count = -1;
long timestamp = System.currentTimeMillis(); long timestamp = System.currentTimeMillis();
int writed_local = 0; int writed_local = 0;
@ -105,10 +159,10 @@ public void downloadFile() throws IOException {
} }
} else { } else {
HttpsURLConnection connect = (HttpsURLConnection) (url).openConnection(); HttpsURLConnection connect = (HttpsURLConnection) (url).openConnection();
connect.setRequestProperty("User-Agent", if (method != null) connect.setRequestMethod(method);
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); // for for (Map.Entry<String, String> ent : requestProps.entrySet()) {
// stupid connect.setRequestProperty(ent.getKey(), ent.getValue());
// servers }
connect.setInstanceFollowRedirects(true); connect.setInstanceFollowRedirects(true);
if (handler != null) if (handler != null)
handler.check(connect.getServerCertificates()); handler.check(connect.getServerCertificates());
@ -116,7 +170,7 @@ public void downloadFile() throws IOException {
throw new IOException(String.format("Invalid response of http server %d.", connect.getResponseCode())); throw new IOException(String.format("Invalid response of http server %d.", connect.getResponseCode()));
try (BufferedInputStream in = new BufferedInputStream(connect.getInputStream(), IOHelper.BUFFER_SIZE); try (BufferedInputStream in = new BufferedInputStream(connect.getInputStream(), IOHelper.BUFFER_SIZE);
FileOutputStream fout = new FileOutputStream(file, skip != 0)) { FileOutputStream fout = new FileOutputStream(file, skip != 0)) {
final byte data[] = new byte[IOHelper.BUFFER_SIZE]; byte data[] = new byte[IOHelper.BUFFER_SIZE];
int count = -1; int count = -1;
long timestamp = System.currentTimeMillis(); long timestamp = System.currentTimeMillis();
int writed_local = 0; int writed_local = 0;