| WebResource.java |
1 /**
2 * Distribution License:
3 * JSword is free software; you can redistribute it and/or modify it under
4 * the terms of the GNU Lesser General Public License, version 2.1 or later
5 * as published by the Free Software Foundation. This program is distributed
6 * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
7 * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
8 * See the GNU Lesser General Public License for more details.
9 *
10 * The License is available on the internet at:
11 * http://www.gnu.org/copyleft/lgpl.html
12 * or by writing to:
13 * Free Software Foundation, Inc.
14 * 59 Temple Place - Suite 330
15 * Boston, MA 02111-1307, USA
16 *
17 * © CrossWire Bible Society, 2005 - 2016
18 *
19 */
20 package org.crosswire.common.util;
21
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.OutputStream;
25 import java.net.URI;
26 import java.util.Date;
27
28 import org.apache.http.Header;
29 import org.apache.http.HttpEntity;
30 import org.apache.http.HttpHost;
31 import org.apache.http.HttpResponse;
32 import org.apache.http.HttpStatus;
33 import org.apache.http.StatusLine;
34 import org.apache.http.client.config.RequestConfig;
35 import org.apache.http.client.methods.HttpGet;
36 import org.apache.http.client.methods.HttpHead;
37 import org.apache.http.client.methods.HttpRequestBase;
38 import org.apache.http.client.utils.DateUtils;
39 import org.apache.http.impl.client.CloseableHttpClient;
40 import org.apache.http.impl.client.HttpClientBuilder;
41 import org.crosswire.common.progress.Progress;
42 import org.crosswire.jsword.JSMsg;
43
44 /**
45 * A WebResource is backed by an URL and potentially the proxy through which it
46 * need go. It can get basic information about the resource and it can get the
47 * resource. The requests are subject to a timeout, which can be set via the
48 * constructor or previously by a call to set the default timeout. The initial
49 * default timeout is 750 milliseconds.
50 *
51 *
52 * @see gnu.lgpl.License The GNU Lesser General Public License for details.
53 * @author DM Smith
54 */
55 public class WebResource {
56 /**
57 * Construct a WebResource for the given URL, while timing out if too much
58 * time has passed.
59 *
60 * @param theURI
61 * the Resource to get via HTTP
62 */
63 public WebResource(URI theURI) {
64 this(theURI, null, null, timeout);
65 }
66
67 /**
68 * Construct a WebResource for the given URL, while timing out if too much
69 * time has passed.
70 *
71 * @param theURI
72 * the Resource to get via HTTP
73 * @param theTimeout
74 * the length of time in milliseconds to allow a connection to
75 * respond before timing out
76 */
77 public WebResource(URI theURI, int theTimeout) {
78 this(theURI, null, null, theTimeout);
79 }
80
81 /**
82 * Construct a WebResource for the given URL, going through the optional
83 * proxy and default port, while timing out if too much time has passed.
84 *
85 * @param theURI
86 * the Resource to get via HTTP
87 * @param theProxyHost
88 * the proxy host or null
89 */
90 public WebResource(URI theURI, String theProxyHost) {
91 this(theURI, theProxyHost, null, timeout);
92 }
93
94 /**
95 * Construct a WebResource for the given URL, going through the optional
96 * proxy and default port, while timing out if too much time has passed.
97 *
98 * @param theURI
99 * the Resource to get via HTTP
100 * @param theProxyHost
101 * the proxy host or null
102 * @param theTimeout
103 * the length of time in milliseconds to allow a connection to
104 * respond before timing out
105 */
106 public WebResource(URI theURI, String theProxyHost, int theTimeout) {
107 this(theURI, theProxyHost, null, theTimeout);
108 }
109
110 /**
111 * Construct a WebResource for the given URL, going through the optional
112 * proxy and port, while timing out if too much time has passed.
113 *
114 * @param theURI
115 * the Resource to get via HTTP
116 * @param theProxyHost
117 * the proxy host or null
118 * @param theProxyPort
119 * the proxy port or null, where null means use the standard port
120 */
121 public WebResource(URI theURI, String theProxyHost, Integer theProxyPort) {
122 this(theURI, theProxyHost, theProxyPort, timeout);
123 }
124
125 /**
126 * Construct a WebResource for the given URL, going through the optional
127 * proxy and port, while timing out if too much time has passed.
128 *
129 * @param theURI
130 * the Resource to get via HTTP
131 * @param theProxyHost
132 * the proxy host or null
133 * @param theProxyPort
134 * the proxy port or null, where null means use the standard port
135 * @param theTimeout
136 * the length of time in milliseconds to allow a connection to
137 * respond before timing out
138 */
139 public WebResource(URI theURI, String theProxyHost, Integer theProxyPort, int theTimeout) {
140 uri = theURI;
141 HttpHost proxy = null;
142
143 // Configure proxy info if necessary and defined
144 if (theProxyHost != null && theProxyHost.length() > 0) {
145 proxy = new HttpHost(theProxyHost, theProxyPort == null ? -1 : theProxyPort.intValue());
146 }
147
148 final RequestConfig.Builder builder = RequestConfig.custom();
149 builder.setConnectTimeout(theTimeout).setConnectionRequestTimeout(theTimeout).setSocketTimeout(theTimeout).setProxy(proxy);
150 client = HttpClientBuilder.create().setDefaultRequestConfig(builder.build()).build();
151 }
152
153 /**
154 * When this WebResource is no longer needed it should be shutdown to return
155 * underlying resources back to the OS.
156 */
157 public void shutdown() {
158 IOUtil.close(client);
159 }
160
161 /**
162 * @return the timeout in milliseconds
163 */
164 public static int getTimeout() {
165 return timeout;
166 }
167
168 /**
169 * @param timeout
170 * the timeout to set in milliseconds
171 */
172 public static void setTimeout(int timeout) {
173 WebResource.timeout = timeout;
174 }
175
176 /**
177 * Determine the size of this WebResource.
178 * <p>
179 * Note that the http client may read the entire file to determine this.
180 * </p>
181 *
182 * @return the size of the file
183 */
184 public int getSize() {
185 HttpRequestBase method = new HttpHead(uri);
186 HttpResponse response = null;
187 try {
188 // Execute the method.
189 response = client.execute(method);
190 StatusLine statusLine = response.getStatusLine();
191 if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
192 return getHeaderAsInt(response, "Content-Length");
193 }
194 String reason = response.getStatusLine().getReasonPhrase();
195 // TRANSLATOR: Common error condition: {0} is a placeholder for the
196 // URL of what could not be found.
197 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath()));
198 } catch (IOException e) {
199 return 0;
200 }
201 return 0;
202 }
203
204 /**
205 * Determine the last modified date of this WebResource.
206 * <p>
207 * Note that the http client may read the entire file.
208 * </p>
209 *
210 * @return the last mod date of the file
211 */
212 public long getLastModified() {
213 HttpRequestBase method = new HttpHead(uri);
214 HttpResponse response = null;
215 try {
216 // Execute the method.
217 response = client.execute(method);
218 StatusLine statusLine = response.getStatusLine();
219 if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
220 return getHeaderAsDate(response, "Last-Modified");
221 }
222 String reason = response.getStatusLine().getReasonPhrase();
223 // TRANSLATOR: Common error condition: {0} is a placeholder for the
224 // URL of what could not be found.
225 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath()));
226 } catch (IOException e) {
227 return new Date().getTime();
228 }
229 return new Date().getTime();
230 }
231
232 /**
233 * Copy this WebResource to the destination and report progress.
234 *
235 * @param dest
236 * the URI of the destination, typically a file:///.
237 * @param meter
238 * the job on which to report progress
239 * @throws LucidException when an error is encountered
240 */
241 public void copy(URI dest, Progress meter) throws LucidException {
242 InputStream in = null;
243 OutputStream out = null;
244 HttpRequestBase method = new HttpGet(uri);
245 HttpResponse response = null;
246 HttpEntity entity = null;
247 try {
248 // Execute the method.
249 response = client.execute(method);
250 // Initialize the meter, if present
251 if (meter != null) {
252 // Find out how big it is
253 int size = getHeaderAsInt(response, "Content-Length");
254 // Sometimes the Content-Length is not given and we have to grab it via HEAD method
255 if (size == 0) {
256 size = getSize();
257 }
258 meter.setTotalWork(size);
259 }
260
261 entity = response.getEntity();
262 if (entity != null) {
263 in = entity.getContent();
264
265 // Download the index file
266 out = NetUtil.getOutputStream(dest);
267
268 byte[] buf = new byte[4096];
269 int count = in.read(buf);
270 while (-1 != count) {
271 if (meter != null) {
272 meter.incrementWorkDone(count);
273 }
274 out.write(buf, 0, count);
275 count = in.read(buf);
276 }
277 } else {
278 String reason = response.getStatusLine().getReasonPhrase();
279 // TRANSLATOR: Common error condition: {0} is a placeholder for
280 // the URL of what could not be found.
281 Reporter.informUser(this, JSMsg.gettext("Unable to find: {0}", reason + ':' + uri.getPath()));
282 }
283 } catch (IOException e) {
284 // TRANSLATOR: Common error condition: {0} is a placeholder for the
285 // URL of what could not be found.
286 throw new LucidException(JSMsg.gettext("Unable to find: {0}", uri.toString()), e);
287 } finally {
288 // Close the streams
289 IOUtil.close(in);
290 IOUtil.close(out);
291 }
292 }
293
294 /**
295 * Copy this WebResource to the destination.
296 *
297 * @param dest the destination URI
298 * @throws LucidException when an error is encountered
299 */
300 public void copy(URI dest) throws LucidException {
301 copy(dest, null);
302 }
303
304 /**
305 * Get the field as a long.
306 *
307 * @param response The response from the request
308 * @param field the header field to check
309 * @return the int value for the field
310 */
311 private int getHeaderAsInt(HttpResponse response, String field) {
312 Header header = response.getFirstHeader(field);
313 // If there is no matching header in the message null is returned.
314 if (header == null) {
315 return 0;
316 }
317
318 String value = header.getValue();
319 try {
320 return Integer.parseInt(value);
321 } catch (NumberFormatException ex) {
322 return 0;
323 }
324 }
325
326 /**
327 * Get the number of seconds since start of epoch for the field in the response headers as a Date.
328 *
329 * @param response The response from the request
330 * @param field the header field to check
331 * @return number of seconds since start of epoch
332 */
333 private long getHeaderAsDate(HttpResponse response, String field) {
334 Header header = response.getFirstHeader(field);
335 String value = header.getValue();
336 // This date cannot be readily parsed with DateFormatter
337 return DateUtils.parseDate(value).getTime();
338 }
339 /**
340 * Define a 750 ms timeout to get a connection
341 */
342 private static int timeout = 750;
343
344 private URI uri;
345 private CloseableHttpClient client;
346 }
347