001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.tools;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedOutputStream;
007import java.io.BufferedReader;
008import java.io.ByteArrayInputStream;
009import java.io.IOException;
010import java.io.InputStream;
011import java.io.OutputStream;
012import java.net.CookieHandler;
013import java.net.CookieManager;
014import java.net.HttpURLConnection;
015import java.net.URL;
016import java.util.Collections;
017import java.util.List;
018import java.util.Locale;
019import java.util.Map;
020import java.util.Map.Entry;
021import java.util.Scanner;
022import java.util.TreeMap;
023import java.util.concurrent.TimeUnit;
024import java.util.regex.Matcher;
025import java.util.regex.Pattern;
026import java.util.zip.GZIPInputStream;
027
028import org.openstreetmap.josm.Main;
029import org.openstreetmap.josm.data.Version;
030import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
031import org.openstreetmap.josm.gui.progress.ProgressMonitor;
032import org.openstreetmap.josm.io.Compression;
033import org.openstreetmap.josm.io.ProgressInputStream;
034import org.openstreetmap.josm.io.ProgressOutputStream;
035import org.openstreetmap.josm.io.UTFInputStreamReader;
036import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
037
038/**
039 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
040 * @since 9168
041 */
042public final class HttpClient {
043
044    private URL url;
045    private final String requestMethod;
046    private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.connect", 15));
047    private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.read", 30));
048    private byte[] requestBody;
049    private long ifModifiedSince;
050    private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
051    private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
052    private boolean useCache;
053    private String reasonForRequest;
054    private HttpURLConnection connection; // to allow disconnecting before `response` is set
055    private Response response;
056    private boolean finishOnCloseOutput = true;
057
058    static {
059        CookieHandler.setDefault(new CookieManager());
060    }
061
062    private HttpClient(URL url, String requestMethod) {
063        this.url = url;
064        this.requestMethod = requestMethod;
065        this.headers.put("Accept-Encoding", "gzip");
066    }
067
068    /**
069     * Opens the HTTP connection.
070     * @return HTTP response
071     * @throws IOException if any I/O error occurs
072     */
073    public Response connect() throws IOException {
074        return connect(null);
075    }
076
077    /**
078     * Opens the HTTP connection.
079     * @param progressMonitor progress monitor
080     * @return HTTP response
081     * @throws IOException if any I/O error occurs
082     * @since 9179
083     */
084    public Response connect(ProgressMonitor progressMonitor) throws IOException {
085        if (progressMonitor == null) {
086            progressMonitor = NullProgressMonitor.INSTANCE;
087        }
088        final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
089        this.connection = connection;
090        connection.setRequestMethod(requestMethod);
091        connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
092        connection.setConnectTimeout(connectTimeout);
093        connection.setReadTimeout(readTimeout);
094        connection.setInstanceFollowRedirects(false); // we do that ourselves
095        if (ifModifiedSince > 0) {
096            connection.setIfModifiedSince(ifModifiedSince);
097        }
098        connection.setUseCaches(useCache);
099        if (!useCache) {
100            connection.setRequestProperty("Cache-Control", "no-cache");
101        }
102        for (Map.Entry<String, String> header : headers.entrySet()) {
103            if (header.getValue() != null) {
104                connection.setRequestProperty(header.getKey(), header.getValue());
105            }
106        }
107
108        progressMonitor.beginTask(tr("Contacting Server..."), 1);
109        progressMonitor.indeterminateSubTask(null);
110
111        if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
112            Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
113            connection.setFixedLengthStreamingMode(requestBody.length);
114            connection.setDoOutput(true);
115            try (OutputStream out = new BufferedOutputStream(
116                    new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) {
117                out.write(requestBody);
118            }
119        }
120
121        boolean successfulConnection = false;
122        try {
123            try {
124                connection.connect();
125                final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
126                Main.info("{0} {1}{2} -> {3}{4}",
127                        requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
128                        connection.getResponseCode(),
129                        connection.getContentLengthLong() > 0
130                                ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
131                                : ""
132                );
133                if (Main.isDebugEnabled()) {
134                    Main.debug("RESPONSE: " + connection.getHeaderFields());
135                }
136                if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
137                    DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
138                }
139            } catch (IOException e) {
140                Main.info("{0} {1} -> !!!", requestMethod, url);
141                Main.warn(e);
142                //noinspection ThrowableResultOfMethodCallIgnored
143                Main.addNetworkError(url, Utils.getRootCause(e));
144                throw e;
145            }
146            if (isRedirect(connection.getResponseCode())) {
147                final String redirectLocation = connection.getHeaderField("Location");
148                if (redirectLocation == null) {
149                    /* I18n: argument is HTTP response code */
150                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
151                            " Can''t redirect. Aborting.", connection.getResponseCode());
152                    throw new IOException(msg);
153                } else if (maxRedirects > 0) {
154                    url = new URL(url, redirectLocation);
155                    maxRedirects--;
156                    Main.info(tr("Download redirected to ''{0}''", redirectLocation));
157                    return connect();
158                } else if (maxRedirects == 0) {
159                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
160                    throw new IOException(msg);
161                }
162            }
163            response = new Response(connection, progressMonitor);
164            successfulConnection = true;
165            return response;
166        } finally {
167            if (!successfulConnection) {
168                connection.disconnect();
169            }
170        }
171    }
172
173    /**
174     * Returns the HTTP response which is set only after calling {@link #connect()}.
175     * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
176     *
177     * @return the HTTP response
178     * @since 9309
179     */
180    public Response getResponse() {
181        return response;
182    }
183
184    /**
185     * A wrapper for the HTTP response.
186     */
187    public static final class Response {
188        private final HttpURLConnection connection;
189        private final ProgressMonitor monitor;
190        private final int responseCode;
191        private final String responseMessage;
192        private boolean uncompress;
193        private boolean uncompressAccordingToContentDisposition;
194        private String responseData;
195
196        private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
197            CheckParameterUtil.ensureParameterNotNull(connection, "connection");
198            CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
199            this.connection = connection;
200            this.monitor = monitor;
201            this.responseCode = connection.getResponseCode();
202            this.responseMessage = connection.getResponseMessage();
203            if (this.responseCode >= 300) {
204                String contentType = getContentType();
205                if (contentType == null || (
206                        contentType.contains("text") ||
207                        contentType.contains("html") ||
208                        contentType.contains("xml"))
209                        ) {
210                    String content = this.fetchContent();
211                    if (content.isEmpty()) {
212                        Main.debug("Server did not return any body");
213                    } else {
214                        Main.debug("Response body: ");
215                        Main.debug(this.fetchContent());
216                    }
217                } else {
218                    Main.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
219                }
220            }
221        }
222
223        /**
224         * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
225         *
226         * @param uncompress whether the input stream should be uncompressed if necessary
227         * @return {@code this}
228         */
229        public Response uncompress(boolean uncompress) {
230            this.uncompress = uncompress;
231            return this;
232        }
233
234        /**
235         * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
236         * HTTP header.
237         * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
238         * {@code Content-Disposition}
239         * @return {@code this}
240         * @since 9172
241         */
242        public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
243            this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
244            return this;
245        }
246
247        /**
248         * Returns the URL.
249         * @return the URL
250         * @see HttpURLConnection#getURL()
251         * @since 9172
252         */
253        public URL getURL() {
254            return connection.getURL();
255        }
256
257        /**
258         * Returns the request method.
259         * @return the HTTP request method
260         * @see HttpURLConnection#getRequestMethod()
261         * @since 9172
262         */
263        public String getRequestMethod() {
264            return connection.getRequestMethod();
265        }
266
267        /**
268         * Returns an input stream that reads from this HTTP connection, or,
269         * error stream if the connection failed but the server sent useful data.
270         * <p>
271         * Note: the return value can be null, if both the input and the error stream are null.
272         * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
273         * @return input or error stream
274         * @throws IOException if any I/O error occurs
275         *
276         * @see HttpURLConnection#getInputStream()
277         * @see HttpURLConnection#getErrorStream()
278         */
279        @SuppressWarnings("resource")
280        public InputStream getContent() throws IOException {
281            InputStream in;
282            try {
283                in = connection.getInputStream();
284            } catch (IOException ioe) {
285                Main.debug(ioe);
286                in = connection.getErrorStream();
287                if (in == null) {
288                    in = new ByteArrayInputStream(new byte[]{});
289                }
290            }
291            in = new ProgressInputStream(in, getContentLength(), monitor);
292            in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
293            Compression compression = Compression.NONE;
294            if (uncompress) {
295                final String contentType = getContentType();
296                Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
297                compression = Compression.forContentType(contentType);
298            }
299            if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
300                final String contentDisposition = getHeaderField("Content-Disposition");
301                final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
302                        contentDisposition != null ? contentDisposition : "");
303                if (matcher.find()) {
304                    Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
305                    compression = Compression.byExtension(matcher.group(1));
306                }
307            }
308            in = compression.getUncompressedInputStream(in);
309            return in;
310        }
311
312        /**
313         * Returns {@link #getContent()} wrapped in a buffered reader.
314         *
315         * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
316         * @return buffered reader
317         * @throws IOException if any I/O error occurs
318         */
319        public BufferedReader getContentReader() throws IOException {
320            return new BufferedReader(
321                    UTFInputStreamReader.create(getContent())
322            );
323        }
324
325        /**
326         * Fetches the HTTP response as String.
327         * @return the response
328         * @throws IOException if any I/O error occurs
329         */
330        public synchronized String fetchContent() throws IOException {
331            if (responseData == null) {
332                try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
333                    responseData = scanner.hasNext() ? scanner.next() : "";
334                }
335            }
336            return responseData;
337        }
338
339        /**
340         * Gets the response code from this HTTP connection.
341         * @return HTTP response code
342         *
343         * @see HttpURLConnection#getResponseCode()
344         */
345        public int getResponseCode() {
346            return responseCode;
347        }
348
349        /**
350         * Gets the response message from this HTTP connection.
351         * @return HTTP response message
352         *
353         * @see HttpURLConnection#getResponseMessage()
354         * @since 9172
355         */
356        public String getResponseMessage() {
357            return responseMessage;
358        }
359
360        /**
361         * Returns the {@code Content-Encoding} header.
362         * @return {@code Content-Encoding} HTTP header
363         * @see HttpURLConnection#getContentEncoding()
364         */
365        public String getContentEncoding() {
366            return connection.getContentEncoding();
367        }
368
369        /**
370         * Returns the {@code Content-Type} header.
371         * @return {@code Content-Type} HTTP header
372         */
373        public String getContentType() {
374            return connection.getHeaderField("Content-Type");
375        }
376
377        /**
378         * Returns the {@code Expire} header.
379         * @return {@code Expire} HTTP header
380         * @see HttpURLConnection#getExpiration()
381         * @since 9232
382         */
383        public long getExpiration() {
384            return connection.getExpiration();
385        }
386
387        /**
388         * Returns the {@code Last-Modified} header.
389         * @return {@code Last-Modified} HTTP header
390         * @see HttpURLConnection#getLastModified()
391         * @since 9232
392         */
393        public long getLastModified() {
394            return connection.getLastModified();
395        }
396
397        /**
398         * Returns the {@code Content-Length} header.
399         * @return {@code Content-Length} HTTP header
400         * @see HttpURLConnection#getContentLengthLong()
401         */
402        public long getContentLength() {
403            return connection.getContentLengthLong();
404        }
405
406        /**
407         * Returns the value of the named header field.
408         * @param name the name of a header field
409         * @return the value of the named header field, or {@code null} if there is no such field in the header
410         * @see HttpURLConnection#getHeaderField(String)
411         * @since 9172
412         */
413        public String getHeaderField(String name) {
414            return connection.getHeaderField(name);
415        }
416
417        /**
418         * Returns an unmodifiable Map mapping header keys to a List of header values.
419         * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
420         * @return unmodifiable Map mapping header keys to a List of header values
421         * @see HttpURLConnection#getHeaderFields()
422         * @since 9232
423         */
424        public Map<String, List<String>> getHeaderFields() {
425            // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
426            Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
427            for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
428                if (e.getKey() != null) {
429                    ret.put(e.getKey(), e.getValue());
430                }
431            }
432            return Collections.unmodifiableMap(ret);
433        }
434
435        /**
436         * @see HttpURLConnection#disconnect()
437         */
438        public void disconnect() {
439            HttpClient.disconnect(connection);
440        }
441    }
442
443    /**
444     * Creates a new instance for the given URL and a {@code GET} request
445     *
446     * @param url the URL
447     * @return a new instance
448     */
449    public static HttpClient create(URL url) {
450        return create(url, "GET");
451    }
452
453    /**
454     * Creates a new instance for the given URL and a {@code GET} request
455     *
456     * @param url the URL
457     * @param requestMethod the HTTP request method to perform when calling
458     * @return a new instance
459     */
460    public static HttpClient create(URL url, String requestMethod) {
461        return new HttpClient(url, requestMethod);
462    }
463
464    /**
465     * Returns the URL set for this connection.
466     * @return the URL
467     * @see #create(URL)
468     * @see #create(URL, String)
469     * @since 9172
470     */
471    public URL getURL() {
472        return url;
473    }
474
475    /**
476     * Returns the request method set for this connection.
477     * @return the HTTP request method
478     * @see #create(URL, String)
479     * @since 9172
480     */
481    public String getRequestMethod() {
482        return requestMethod;
483    }
484
485    /**
486     * Returns the set value for the given {@code header}.
487     * @param header HTTP header name
488     * @return HTTP header value
489     * @since 9172
490     */
491    public String getRequestHeader(String header) {
492        return headers.get(header);
493    }
494
495    /**
496     * Sets whether not to set header {@code Cache-Control=no-cache}
497     *
498     * @param useCache whether not to set header {@code Cache-Control=no-cache}
499     * @return {@code this}
500     * @see HttpURLConnection#setUseCaches(boolean)
501     */
502    public HttpClient useCache(boolean useCache) {
503        this.useCache = useCache;
504        return this;
505    }
506
507    /**
508     * Sets whether not to set header {@code Connection=close}
509     * <p>
510     * This might fix #7640, see
511     * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
512     *
513     * @param keepAlive whether not to set header {@code Connection=close}
514     * @return {@code this}
515     */
516    public HttpClient keepAlive(boolean keepAlive) {
517        return setHeader("Connection", keepAlive ? null : "close");
518    }
519
520    /**
521     * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
522     * by this URLConnection. If the timeout expires before the connection can be established, a
523     * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
524     * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
525     * @return {@code this}
526     * @see HttpURLConnection#setConnectTimeout(int)
527     */
528    public HttpClient setConnectTimeout(int connectTimeout) {
529        this.connectTimeout = connectTimeout;
530        return this;
531    }
532
533    /**
534     * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
535     * input stream when a connection is established to a resource. If the timeout expires before there is data available for
536     * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
537     * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
538     * @return {@code this}
539     * @see HttpURLConnection#setReadTimeout(int)
540     */
541    public HttpClient setReadTimeout(int readTimeout) {
542        this.readTimeout = readTimeout;
543        return this;
544    }
545
546    /**
547     * Sets the {@code Accept} header.
548     * @param accept header value
549     *
550     * @return {@code this}
551     */
552    public HttpClient setAccept(String accept) {
553        return setHeader("Accept", accept);
554    }
555
556    /**
557     * Sets the request body for {@code PUT}/{@code POST} requests.
558     * @param requestBody request body
559     *
560     * @return {@code this}
561     */
562    public HttpClient setRequestBody(byte[] requestBody) {
563        this.requestBody = requestBody;
564        return this;
565    }
566
567    /**
568     * Sets the {@code If-Modified-Since} header.
569     * @param ifModifiedSince header value
570     *
571     * @return {@code this}
572     */
573    public HttpClient setIfModifiedSince(long ifModifiedSince) {
574        this.ifModifiedSince = ifModifiedSince;
575        return this;
576    }
577
578    /**
579     * Sets the maximum number of redirections to follow.
580     *
581     * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
582     * to not throw an {@link IOException} in {@link #connect()}.
583     * @param maxRedirects header value
584     *
585     * @return {@code this}
586     */
587    public HttpClient setMaxRedirects(int maxRedirects) {
588        this.maxRedirects = maxRedirects;
589        return this;
590    }
591
592    /**
593     * Sets an arbitrary HTTP header.
594     * @param key header name
595     * @param value header value
596     *
597     * @return {@code this}
598     */
599    public HttpClient setHeader(String key, String value) {
600        this.headers.put(key, value);
601        return this;
602    }
603
604    /**
605     * Sets arbitrary HTTP headers.
606     * @param headers HTTP headers
607     *
608     * @return {@code this}
609     */
610    public HttpClient setHeaders(Map<String, String> headers) {
611        this.headers.putAll(headers);
612        return this;
613    }
614
615    /**
616     * Sets a reason to show on console. Can be {@code null} if no reason is given.
617     * @param reasonForRequest Reason to show
618     * @return {@code this}
619     * @since 9172
620     */
621    public HttpClient setReasonForRequest(String reasonForRequest) {
622        this.reasonForRequest = reasonForRequest;
623        return this;
624    }
625
626    /**
627     * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
628     * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
629     * @return {@code this}
630     * @since 10302
631     */
632    public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
633        this.finishOnCloseOutput = finishOnCloseOutput;
634        return this;
635    }
636
637    private static boolean isRedirect(final int statusCode) {
638        switch (statusCode) {
639            case HttpURLConnection.HTTP_MOVED_PERM: // 301
640            case HttpURLConnection.HTTP_MOVED_TEMP: // 302
641            case HttpURLConnection.HTTP_SEE_OTHER: // 303
642            case 307: // TEMPORARY_REDIRECT:
643            case 308: // PERMANENT_REDIRECT:
644                return true;
645            default:
646                return false;
647        }
648    }
649
650    /**
651     * @see HttpURLConnection#disconnect()
652     * @since 9309
653     */
654    public void disconnect() {
655        HttpClient.disconnect(connection);
656    }
657
658    private static void disconnect(final HttpURLConnection connection) {
659        // Fix upload aborts - see #263
660        connection.setConnectTimeout(100);
661        connection.setReadTimeout(100);
662        try {
663            Thread.sleep(100);
664        } catch (InterruptedException ex) {
665            Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
666        }
667        connection.disconnect();
668    }
669}