001: /* *****************************************************************************
002: * HTTPDataSource.java
003: * ****************************************************************************/
004:
005: /* J_LZ_COPYRIGHT_BEGIN *******************************************************
006: * Copyright 2001-2008 Laszlo Systems, Inc. All Rights Reserved. *
007: * Use is subject to license terms. *
008: * J_LZ_COPYRIGHT_END *********************************************************/
009:
010: package org.openlaszlo.data;
011:
012: import java.util.Enumeration;
013: import java.util.Hashtable;
014: import java.util.StringTokenizer;
015: import java.net.URL;
016: import java.net.URLDecoder;
017: import java.net.MalformedURLException;
018: import java.net.UnknownHostException;
019: import java.io.*;
020: import javax.servlet.http.HttpServletRequest;
021: import javax.servlet.http.HttpServletResponse;
022:
023: import org.apache.commons.httpclient.*;
024: import org.apache.commons.httpclient.methods.*;
025: import org.apache.commons.httpclient.util.*;
026: import org.apache.log4j.*;
027:
028: import org.openlaszlo.xml.internal.XMLUtils;
029: import org.openlaszlo.utils.LZHttpUtils;
030: import org.openlaszlo.utils.LZGetMethod;
031: import org.openlaszlo.utils.LZPostMethod;
032: import org.openlaszlo.utils.LZPutMethod;
033: import org.openlaszlo.utils.LZDeleteMethod;
034: import org.openlaszlo.utils.FileUtils;
035: import org.openlaszlo.server.LPS;
036: import org.apache.oro.text.regex.*;
037:
038: /**
039: * HTTP Transport
040: */
041: public class HTTPDataSource extends DataSource {
042:
043: private static Logger mLogger = Logger
044: .getLogger(HTTPDataSource.class);
045:
046: /** Connection Manager */
047: private static MultiThreadedHttpConnectionManager mConnectionMgr = null;
048:
049: /** max number of http retries */
050: private static int mMaxRetries = 1;
051:
052: /** Whether or not to use the http11 . */
053: private static boolean mUseHttp11 = true;
054:
055: /** Connection timeout millis (0 means default) */
056: private static int mConnectionTimeout = 0;
057:
058: /** Timeout millis (0 means infinity) */
059: private static int mTimeout = 0;
060:
061: /** Connection pool timeout in millis (0 means infinity) */
062: private static int mConnectionPoolTimeout = 0;
063:
064: /** Max total connections. */
065: private static int mMaxTotalConnections = 1000;
066:
067: /** Max connections per host. */
068: private static int mMaxConnectionsPerHost = mMaxTotalConnections;
069:
070: /** Number of backend redirects we allow (potential security hole) */
071: private static int mFollowRedirects = 0;
072:
073: {
074: String useMultiThreadedConnectionMgr = LPS.getProperty(
075: "http.useConnectionPool", "true");
076:
077: if (Boolean.valueOf(useMultiThreadedConnectionMgr)
078: .booleanValue()) {
079: mLogger.info(
080: /* (non-Javadoc)
081: * @i18n.test
082: * @org-mes="using connection pool"
083: */
084: org.openlaszlo.i18n.LaszloMessages.getMessage(
085: HTTPDataSource.class.getName(), "051018-81"));
086: mConnectionMgr = new MultiThreadedHttpConnectionManager();
087: } else {
088: mLogger.info(
089: /* (non-Javadoc)
090: * @i18n.test
091: * @org-mes="not using connection pool"
092: */
093: org.openlaszlo.i18n.LaszloMessages.getMessage(
094: HTTPDataSource.class.getName(), "051018-91"));
095: }
096:
097: // Parse multi connection properties anyway. May be used by AXIS. See
098: // ResponderCache for details.
099: {
100: String maxConns = LPS.getProperty("http.maxConns", "1000");
101: mMaxTotalConnections = Integer.parseInt(maxConns);
102: if (mConnectionMgr != null) {
103: mConnectionMgr
104: .setMaxTotalConnections(mMaxTotalConnections);
105: }
106:
107: maxConns = LPS
108: .getProperty("http.maxConnsPerHost", maxConns);
109: mMaxConnectionsPerHost = Integer.parseInt(maxConns);
110: if (mConnectionMgr != null) {
111: mConnectionMgr
112: .setMaxConnectionsPerHost(mMaxConnectionsPerHost);
113: }
114: }
115:
116: String maxRetries = LPS.getProperty("http.maxBackendRetries",
117: "1");
118: mMaxRetries = Integer.parseInt(maxRetries);
119:
120: String followRedirects = LPS.getProperty(
121: "http.followRedirects", "0");
122: mFollowRedirects = Integer.parseInt(followRedirects);
123:
124: String timeout = LPS
125: .getProperty("http.backendTimeout", "30000");
126: mTimeout = Integer.parseInt(timeout);
127:
128: timeout = LPS.getProperty("http.backendConnectionTimeout",
129: timeout);
130: mConnectionTimeout = Integer.parseInt(timeout);
131:
132: timeout = LPS.getProperty("http.connectionPoolTimeout", "0");
133: mConnectionPoolTimeout = Integer.parseInt(timeout);
134:
135: String useHttp11 = LPS.getProperty("http.useHttp11", "true");
136: mUseHttp11 = Boolean.valueOf(useHttp11).booleanValue();
137: if (mUseHttp11) {
138: mLogger.info("using HTTP 1.1");
139: } else {
140: mLogger.info("not using HTTP 1.1");
141: }
142: }
143:
144: /**
145: * @return name of this datasource
146: */
147: public String name() {
148: return "http";
149: }
150:
151: /**
152: * Do an HTTP Get/Post based on this request
153: *
154: * @return the data from this request
155: * @param app absolute pathnane to app file
156: * @param req request in progress (possibly null)
157: * @param since this is the timestamp on the
158: * currently cached item; this time can be used as the datasource
159: * sees fit (or ignored) in constructing the results. If
160: * the value is -1, assume there is no currently cached item.
161: */
162: public Data getData(String app, HttpServletRequest req,
163: HttpServletResponse res, long since)
164: throws DataSourceException, IOException {
165: return getHTTPData(req, res, getURL(req), since);
166: }
167:
168: public static Data getHTTPData(HttpServletRequest req,
169: HttpServletResponse res, String surl, long since)
170: throws DataSourceException, IOException {
171:
172: int tries = 1;
173:
174: // timeout msecs of time we're allowed in this routine
175: // we must return or throw an exception. 0 means infinite.
176: int timeout = mTimeout;
177: if (req != null) {
178: String timeoutParm = req.getParameter("timeout");
179: if (timeoutParm != null) {
180: timeout = Integer.parseInt(timeoutParm);
181: }
182: }
183:
184: long t1 = System.currentTimeMillis();
185: long elapsed = 0;
186: if (surl == null) {
187: surl = getURL(req);
188: }
189:
190: while (true) {
191: String m = null;
192:
193: long tout;
194: if (timeout > 0) {
195: tout = timeout - elapsed;
196: if (tout <= 0) {
197: throw new InterruptedIOException(
198: /* (non-Javadoc)
199: * @i18n.test
200: * @org-mes=p[0] + " timed out"
201: */
202: org.openlaszlo.i18n.LaszloMessages.getMessage(
203: HTTPDataSource.class.getName(),
204: "051018-194", new Object[] { surl }));
205: }
206: } else {
207: tout = 0;
208: }
209:
210: try {
211: HttpData data = getDataOnce(req, res, since, surl, 0,
212: (int) tout);
213: if (data.code >= 400) {
214: data.release();
215: throw new DataSourceException(
216: errorMessage(data.code));
217: }
218: return data;
219: } catch (HttpRecoverableException e) {
220: // This type of exception should be retried.
221: if (tries++ > mMaxRetries) {
222: throw new InterruptedIOException(
223: /* (non-Javadoc)
224: * @i18n.test
225: * @org-mes="too many retries, exception: " + p[0]
226: */
227: org.openlaszlo.i18n.LaszloMessages.getMessage(
228: HTTPDataSource.class.getName(),
229: "051018-217",
230: new Object[] { e.getMessage() }));
231: }
232: mLogger.warn(
233: /* (non-Javadoc)
234: * @i18n.test
235: * @org-mes="retrying a recoverable exception: " + p[0]
236: */
237: org.openlaszlo.i18n.LaszloMessages.getMessage(
238: HTTPDataSource.class.getName(), "051018-226",
239: new Object[] { e.getMessage() }));
240: } catch (HttpException e) {
241: String msg =
242: /* (non-Javadoc)
243: * @i18n.test
244: * @org-mes="HttpException: " + p[0]
245: */
246: org.openlaszlo.i18n.LaszloMessages.getMessage(
247: HTTPDataSource.class.getName(), "051018-235",
248: new Object[] { e.getMessage() });
249: throw new IOException(
250: /* (non-Javadoc)
251: * @i18n.test
252: * @org-mes="HttpException: " + p[0]
253: */
254: org.openlaszlo.i18n.LaszloMessages.getMessage(
255: HTTPDataSource.class.getName(), "051018-235",
256: new Object[] { e.getMessage() }));
257: } catch (IOException e) {
258:
259: try {
260: Class ssle = Class
261: .forName("javax.net.ssl.SSLException");
262: if (ssle.isAssignableFrom(e.getClass())) {
263: throw new DataSourceException(
264: /* (non-Javadoc)
265: * @i18n.test
266: * @org-mes="SSL exception: " + p[0]
267: */
268: org.openlaszlo.i18n.LaszloMessages.getMessage(
269: HTTPDataSource.class.getName(),
270: "051018-256", new Object[] { e
271: .getMessage() }));
272: }
273: } catch (ClassNotFoundException cfne) {
274: }
275:
276: throw e;
277: }
278:
279: long t2 = System.currentTimeMillis();
280: elapsed = (t2 - t1);
281: }
282: }
283:
284: /**
285: * convenience routine missing from http library
286: */
287: static boolean isRedirect(int rc) {
288: return (rc == HttpStatus.SC_MOVED_PERMANENTLY
289: || rc == HttpStatus.SC_MOVED_TEMPORARILY
290: || rc == HttpStatus.SC_SEE_OTHER || rc == HttpStatus.SC_TEMPORARY_REDIRECT);
291: }
292:
293: /**
294: * @param since last modified time to use
295: * @param req
296: * @param url if null, ignored
297: * @param redirCount number of redirs we've done
298: */
299: public static HttpData getDataOnce(HttpServletRequest req,
300: HttpServletResponse res, long since, String surl,
301: int redirCount, int timeout) throws IOException,
302: HttpException, DataSourceException, MalformedURLException {
303:
304: HttpMethodBase request = null;
305: HostConfiguration hcfg = new HostConfiguration();
306:
307: /*
308: [todo hqm 2006-02-01] Anyone know why this code was here? It is setting
309: the mime type to something which just confuses the DHTML parser.
310:
311: if (res != null) {
312: res.setContentType("application/x-www-form-urlencoded;charset=UTF-8");
313: }
314: */
315:
316: try {
317:
318: // TODO: [2002-01-09 bloch] cope with cache-control
319: // response headers (no-store, no-cache, must-revalidate,
320: // proxy-revalidate).
321:
322: if (surl == null) {
323: surl = getURL(req);
324: }
325: if (surl == null || surl.equals("")) {
326: throw new MalformedURLException(
327: /* (non-Javadoc)
328: * @i18n.test
329: * @org-mes="url is empty or null"
330: */
331: org.openlaszlo.i18n.LaszloMessages.getMessage(
332: HTTPDataSource.class.getName(), "051018-312"));
333: }
334:
335: String reqType = "";
336: String headers = "";
337:
338: if (req != null) {
339: reqType = req.getParameter("reqtype");
340: headers = req.getParameter("headers");
341: }
342:
343: boolean isPost = false;
344: mLogger.debug("reqtype = " + reqType);
345:
346: if (reqType != null && reqType.equals("POST")) {
347: request = new LZPostMethod();
348: request
349: .setRequestHeader("Content-Type",
350: "application/x-www-form-urlencoded;charset=UTF-8");
351: isPost = true;
352: mLogger.debug("setting POST req method");
353: } else if (reqType != null && reqType.equals("PUT")) {
354: request = new LZPutMethod();
355: // todo [hqm 2007] treat PUT like POST?
356: isPost = true;
357: mLogger.debug("setting PUT req method");
358: } else if (reqType != null && reqType.equals("DELETE")) {
359: request = new LZDeleteMethod();
360: mLogger.debug("setting DELETE req method");
361: } else {
362: mLogger.debug("setting GET (default) req method");
363: request = new LZGetMethod();
364: }
365:
366: request.setHttp11(mUseHttp11);
367:
368: // Proxy the request headers
369: if (req != null) {
370: LZHttpUtils.proxyRequestHeaders(req, request);
371: }
372:
373: // Set headers from query string
374: if (headers != null && headers.length() > 0) {
375: StringTokenizer st = new StringTokenizer(headers, "\n");
376: while (st.hasMoreTokens()) {
377: String h = st.nextToken();
378: int i = h.indexOf(":");
379: if (i > -1) {
380: String n = h.substring(0, i);
381: String v = h.substring(i + 2, h.length());
382: request.setRequestHeader(n, v);
383: mLogger.debug(
384: /* (non-Javadoc)
385: * @i18n.test
386: * @org-mes="setting header " + p[0] + "=" + p[1]
387: */
388: org.openlaszlo.i18n.LaszloMessages.getMessage(
389: HTTPDataSource.class.getName(),
390: "051018-359", new Object[] { n, v }));
391: }
392: }
393: }
394:
395: mLogger.debug("Parsing url");
396: URI uri = LZHttpUtils.newURI(surl);
397: try {
398: hcfg.setHost(uri);
399: } catch (Exception e) {
400: throw new MalformedURLException(
401: /* (non-Javadoc)
402: * @i18n.test
403: * @org-mes="can't form uri from " + p[0]
404: */
405: org.openlaszlo.i18n.LaszloMessages.getMessage(
406: HTTPDataSource.class.getName(), "051018-376",
407: new Object[] { surl }));
408: }
409:
410: // This gets us the url-encoded (escaped) path and query string
411: String path = uri.getEscapedPath();
412: String query = uri.getEscapedQuery();
413: mLogger.debug(
414: /* (non-Javadoc)
415: * @i18n.test
416: * @org-mes="encoded path: " + p[0]
417: */
418: org.openlaszlo.i18n.LaszloMessages.getMessage(
419: HTTPDataSource.class.getName(), "051018-389",
420: new Object[] { path }));
421: mLogger.debug(
422: /* (non-Javadoc)
423: * @i18n.test
424: * @org-mes="encoded query: " + p[0]
425: */
426: org.openlaszlo.i18n.LaszloMessages.getMessage(
427: HTTPDataSource.class.getName(), "051018-397",
428: new Object[] { query }));
429:
430: // This call takes a decoded (unescaped) path
431: request.setPath(path);
432:
433: boolean hasQuery = (query != null && query.length() > 0);
434:
435: if (isPost) {
436: String postbody = req.getParameter("lzpostbody");
437: // If there is a lzpostbody arg, use it as the POST request body,
438: // and copy the query arg from the client-supplied URL to the proxy request URL
439: if (postbody != null) {
440: ((EntityEnclosingMethod) request)
441: .setRequestBody(postbody);
442: request.setQueryString(query);
443: } else if (hasQuery) {
444: StringTokenizer st = new StringTokenizer(query, "&");
445: while (st.hasMoreTokens()) {
446: String it = st.nextToken();
447: int i = it.indexOf("=");
448: if (i > 0) {
449: String n = it.substring(0, i);
450: String v = it.substring(i + 1, it.length());
451: // POST encodes values during request
452: ((PostMethod) request).addParameter(n,
453: URLDecoder.decode(v, "UTF-8"));
454: } else {
455: mLogger.warn(
456: /* (non-Javadoc)
457: * @i18n.test
458: * @org-mes="ignoring bad token (missing '=' char) in query string: " + p[0]
459: */
460: org.openlaszlo.i18n.LaszloMessages
461: .getMessage(HTTPDataSource.class
462: .getName(), "051018-429",
463: new Object[] { it }));
464: }
465: }
466: }
467: } else {
468: // This call takes an encoded (escaped) query string
469: request.setQueryString(query);
470: }
471:
472: // Put in the If-Modified-Since headers
473: if (since != -1) {
474: String lms = LZHttpUtils.getDateString(since);
475: request.setRequestHeader(LZHttpUtils.IF_MODIFIED_SINCE,
476: lms);
477: mLogger.debug(
478: /* (non-Javadoc)
479: * @i18n.test
480: * @org-mes="proxying lms: " + p[0]
481: */
482: org.openlaszlo.i18n.LaszloMessages.getMessage(
483: HTTPDataSource.class.getName(), "051018-450",
484: new Object[] { lms }));
485: }
486:
487: mLogger.debug(
488: /* (non-Javadoc)
489: * @i18n.test
490: * @org-mes="setting up http client"
491: */
492: org.openlaszlo.i18n.LaszloMessages.getMessage(
493: HTTPDataSource.class.getName(), "051018-460"));
494: HttpClient htc = null;
495: if (mConnectionMgr != null) {
496: htc = new HttpClient(mConnectionMgr);
497: } else {
498: htc = new HttpClient();
499: }
500:
501: htc.setHostConfiguration(hcfg);
502:
503: // This is the data timeout
504: mLogger.debug(
505: /* (non-Javadoc)
506: * @i18n.test
507: * @org-mes="timeout set to " + p[0]
508: */
509: org.openlaszlo.i18n.LaszloMessages.getMessage(
510: HTTPDataSource.class.getName(), "051018-478",
511: new Object[] { new Integer(timeout) }));
512: htc.setTimeout(timeout);
513:
514: // Set connection timeout the same
515: htc.setConnectionTimeout(mConnectionTimeout);
516:
517: // Set timeout for getting a connection
518: htc.setHttpConnectionFactoryTimeout(mConnectionPoolTimeout);
519:
520: // TODO: [2003-03-05 bloch] this should be more configurable (per app?)
521: if (!isPost) {
522: request.setFollowRedirects(mFollowRedirects > 0);
523: }
524:
525: long t1 = System.currentTimeMillis();
526: mLogger.debug("starting remote request");
527: int rc = htc.executeMethod(hcfg, request);
528: String status = HttpStatus.getStatusText(rc);
529: if (status == null) {
530: status = "" + rc;
531: }
532: mLogger.debug(
533: /* (non-Javadoc)
534: * @i18n.test
535: * @org-mes="remote response status: " + p[0]
536: */
537: org.openlaszlo.i18n.LaszloMessages.getMessage(
538: HTTPDataSource.class.getName(), "051018-504",
539: new Object[] { status }));
540:
541: HttpData data = null;
542: if (isRedirect(rc) && mFollowRedirects > redirCount) {
543: String loc = request.getResponseHeader("Location")
544: .toString();
545: String hostURI = loc.substring(loc.indexOf(": ") + 2,
546: loc.length());
547: mLogger.info(
548: /* (non-Javadoc)
549: * @i18n.test
550: * @org-mes="Following URL from redirect: " + p[0]
551: */
552: org.openlaszlo.i18n.LaszloMessages.getMessage(
553: HTTPDataSource.class.getName(), "051018-517",
554: new Object[] { hostURI }));
555: long t2 = System.currentTimeMillis();
556: if (timeout > 0) {
557: timeout -= (t2 - t1);
558: if (timeout < 0) {
559: throw new InterruptedIOException(
560: /* (non-Javadoc)
561: * @i18n.test
562: * @org-mes=p[0] + " timed out after redirecting to " + p[1]
563: */
564: org.openlaszlo.i18n.LaszloMessages.getMessage(
565: HTTPDataSource.class.getName(),
566: "051018-529",
567: new Object[] { surl, loc }));
568: }
569: }
570:
571: data = getDataOnce(req, res, since, hostURI,
572: redirCount++, timeout);
573: } else {
574: data = new HttpData(request, rc);
575: }
576:
577: if (req != null && res != null) {
578: // proxy response headers
579: LZHttpUtils.proxyResponseHeaders(request, res, req
580: .isSecure());
581: }
582:
583: return data;
584:
585: } catch (ConnectTimeoutException ce) {
586: // Transduce to an InterrupedIOException, since lps takes these to be timeouts.
587: if (request != null) {
588: request.releaseConnection();
589: }
590: throw new InterruptedIOException(
591: /* (non-Javadoc)
592: * @i18n.test
593: * @org-mes="connecting to " + p[0] + ":" + p[1] + " timed out beyond " + p[2] + " msecs."
594: */
595: org.openlaszlo.i18n.LaszloMessages.getMessage(
596: HTTPDataSource.class.getName(), "051018-557",
597: new Object[] { hcfg.getHost(),
598: new Integer(hcfg.getPort()),
599: new Integer(mConnectionTimeout) }));
600: } catch (HttpRecoverableException hre) {
601: if (request != null) {
602: request.releaseConnection();
603: }
604: throw hre;
605: } catch (HttpException e) {
606: if (request != null) {
607: request.releaseConnection();
608: }
609: throw e;
610: } catch (IOException ie) {
611: if (request != null) {
612: request.releaseConnection();
613: }
614: throw ie;
615: } catch (RuntimeException e) {
616: if (request != null) {
617: request.releaseConnection();
618: }
619: throw e;
620: }
621: }
622:
623: /**
624: * utility
625: */
626: private static String errorMessage(int code) {
627: return
628: /* (non-Javadoc)
629: * @i18n.test
630: * @org-mes="HTTP Status code: " + p[0] + ":" + p[1]
631: */
632: org.openlaszlo.i18n.LaszloMessages.getMessage(
633: HTTPDataSource.class.getName(), "051018-592",
634: new Object[] { new Integer(code),
635: HttpStatus.getStatusText(code) });
636: }
637:
638: public static int getConnectionPoolTimeout() {
639: return mConnectionPoolTimeout;
640: }
641:
642: public static int getMaxTotalConnections() {
643: return mMaxTotalConnections;
644: }
645:
646: public static int getMaxConnectionsPerHost() {
647: return mMaxConnectionsPerHost;
648: }
649:
650: public static void main(String args[]) {
651:
652: HTTPDataSource ds = new HTTPDataSource();
653:
654: try {
655: if (args.length != 1 && args.length != 2) {
656: throw new Exception(
657: /* (non-Javadoc)
658: * @i18n.test
659: * @org-mes="Need an url"
660: */
661: org.openlaszlo.i18n.LaszloMessages.getMessage(
662: HTTPDataSource.class.getName(), "051018-828"));
663: }
664: String surl = args[0];
665: FileOutputStream out = null;
666: if (args.length == 2) {
667: out = new FileOutputStream(args[1]);
668: }
669: System.out.println("url is " + surl);
670:
671: HttpData data = ds.getDataOnce(null, null, -1, surl, 0, 0);
672:
673: System.out.println("Response code: " + data.code);
674:
675: if (out != null) {
676: FileUtils.send(data.getInputStream(), out);
677: }
678:
679: data.release();
680:
681: } catch (Exception e) {
682: e.printStackTrace();
683: }
684: }
685: }
|