001: /* TransclusionDecideRule
002: *
003: * $Id: TransclusionDecideRule.java 4952 2007-03-03 01:31:56Z gojomo $
004: *
005: * Created on Apr 1, 2005
006: *
007: * Copyright (C) 2005 Internet Archive.
008: *
009: * This file is part of the Heritrix web crawler (crawler.archive.org).
010: *
011: * Heritrix is free software; you can redistribute it and/or modify
012: * it under the terms of the GNU Lesser Public License as published by
013: * the Free Software Foundation; either version 2.1 of the License, or
014: * any later version.
015: *
016: * Heritrix is distributed in the hope that it will be useful,
017: * but WITHOUT ANY WARRANTY; without even the implied warranty of
018: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: * GNU Lesser Public License for more details.
020: *
021: * You should have received a copy of the GNU Lesser Public License
022: * along with Heritrix; if not, write to the Free Software
023: * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: */
025: package org.archive.crawler.deciderules;
026:
027: import org.archive.crawler.datamodel.CandidateURI;
028: import org.archive.crawler.extractor.Link;
029: import org.archive.crawler.settings.SimpleType;
030: import org.archive.crawler.settings.Type;
031:
032: /**
033: * Rule ACCEPTs any CrawlURIs whose path-from-seed ('hopsPath' -- see
034: * {@link CandidateURI#getPathFromSeed()}) ends
035: * with at least one, but not more than, the given number of
036: * non-navlink ('L') hops.
037: *
038: * Otherwise, if the path-from-seed is empty or if a navlink ('L') occurs
039: * within max-trans-hops of the tail of the path-from-seed, this rule
040: * returns PASS.
041: *
042: * <p>Thus, it allows things like embedded resources (frames/images/media)
043: * and redirects to be transitively included ('transcluded') in a crawl,
044: * even if they otherwise would not, for some reasonable number of hops
045: * (1-4).
046: *
047: * @see <a href="http://www.google.com/search?q=define%3Atransclusion&sourceid=mozilla&start=0&start=0&ie=utf-8&oe=utf-8">Transclusion</a>
048: *
049: * @author gojomo
050: */
051: public class TransclusionDecideRule extends PredicatedDecideRule {
052:
053: private static final long serialVersionUID = -3975688876990558918L;
054:
055: private static final String ATTR_MAX_TRANS_HOPS = "max-trans-hops";
056:
057: private static final String ATTR_MAX_SPECULATIVE_HOPS = "max-speculative-hops";
058:
059: /**
060: * Default maximum transitive hops -- any type
061: * Default access so can be accessed by unit tests.
062: */
063: static final Integer DEFAULT_MAX_TRANS_HOPS = new Integer(3);
064:
065: /**
066: * Default maximum speculative ('X') hops.
067: * Default access so can be accessed by unit tests.
068: */
069: static final Integer DEFAULT_MAX_SPECULATIVE_HOPS = new Integer(1);
070:
071: /**
072: * Usual constructor.
073: * @param name Name of this DecideRule.
074: */
075: public TransclusionDecideRule(String name) {
076: super (name);
077: setDescription("TransclusionDecideRule. ACCEPTs URIs whose path "
078: + "from the seed ends with up to (but not more than) the "
079: + "configured '"
080: + ATTR_MAX_TRANS_HOPS
081: + "' number of non-navlink ('L') hops.");
082: // make default ACCEPT unchangeable
083: Type type = getElementFromDefinition(ATTR_DECISION);
084: type.setTransient(true);
085: addElementToDefinition(new SimpleType(
086: ATTR_MAX_TRANS_HOPS,
087: "Maximum number of non-navlink (non-'L') hops to ACCEPT.",
088: DEFAULT_MAX_TRANS_HOPS));
089: addElementToDefinition(new SimpleType(
090: ATTR_MAX_SPECULATIVE_HOPS,
091: "Maximum number of speculative ('X') hops to ACCEPT.",
092: DEFAULT_MAX_SPECULATIVE_HOPS));
093: }
094:
095: /**
096: * Evaluate whether given object is within the threshold number of
097: * transitive hops.
098: *
099: * @param object Object to make decision on.
100: * @return true if the transitive hops >0 and <= max
101: */
102: protected boolean evaluate(Object object) {
103: CandidateURI curi = null;
104: try {
105: curi = (CandidateURI) object;
106: } catch (ClassCastException e) {
107: // if not CrawlURI, always disregard.
108: return false;
109: }
110: String hopsPath = curi.getPathFromSeed();
111: if (hopsPath == null || hopsPath.length() == 0) {
112: return false;
113: }
114: int count = 0;
115: int specCount = 0;
116: for (int i = hopsPath.length() - 1; i >= 0; i--) {
117: char c = hopsPath.charAt(i);
118: if (c != Link.NAVLINK_HOP) {
119: count++;
120: if (c == Link.SPECULATIVE_HOP) {
121: specCount++;
122: }
123: } else {
124: break;
125: }
126: }
127: return count > 0
128: && (specCount <= getThresholdSpeculativeHops(object) && count <= getThresholdHops(object));
129: }
130:
131: /**
132: * @param obj Context object.
133: * @return hops cutoff threshold
134: */
135: private int getThresholdHops(Object obj) {
136: return ((Integer) getUncheckedAttribute(obj,
137: ATTR_MAX_TRANS_HOPS)).intValue();
138: }
139:
140: /**
141: * @param obj Context object.
142: * @return hops cutoff threshold
143: */
144: private int getThresholdSpeculativeHops(Object obj) {
145: return ((Integer) getUncheckedAttribute(obj,
146: ATTR_MAX_SPECULATIVE_HOPS)).intValue();
147: }
148: }
|