Package com.crawljax.core

Class Crawler

java.lang.Object

com.crawljax.core.Crawler

public class Crawler
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
private boolean	avoidDifferentBacktracking
private boolean	avoidUnrelatedBacktracking
private static long	BACKTRACKING_SEED
private URI	basicAuthUrl
private EmbeddedBrowser	browser
private UnfiredFragmentCandidates	candidateActionCache
private CandidateElementExtractor	candidateExtractor
private CrawlerContext	context
private AtomicInteger	crawlDepth
private CrawlPath	crawlpath
private CrawlRules	crawlRules
private CrawlScope	crawlScope
private static int	DUPLICATE_EVENT_SEED
private FormHandler	formHandler
private FragmentManager	fragmentManager
private javax.inject.Provider<InMemoryStateFlowGraph>	graphProvider
private static org.slf4j.Logger	LOG
private int	maxDepth
private long	nextBackTrackingId
private long	nextEventableId
private boolean	pairwiseFormHandling
private Plugins	plugins
private StateComparator	stateComparator
private StateMachine	stateMachine
private URI	url
private boolean	UseEquivalentReset
private StateVertexFactory	vertexFactory
private WaitConditionChecker	waitConditionChecker

Constructor Summary

Constructors

Constructor

Description

Crawler(CrawlerContext context, CrawljaxConfiguration config, StateComparator stateComparator, UnfiredFragmentCandidates candidateActionCache, CoreModule.FormHandlerFactory formHandlerFactory, CoreModule.TrainingFormHandlerFactory trainingFormHandlerFactory, WaitConditionChecker waitConditionChecker, CoreModule.CandidateElementExtractorFactory elementExtractor, javax.inject.Provider<InMemoryStateFlowGraph> graphProvider, Plugins plugins, StateVertexFactory vertexFactory)

Method Summary

Modifier and Type	Method	Description
private static void	addPathInfo(CrawlPath path, StringBuilder builder)
private StateVertex	changeState(StateVertex targetState, Eventable clickable)
private void	checkCrawlConditions(StateVertex targetState)
private void	checkOnURLState(StateVertex previousState)
void	close()	Close the browser.
private boolean	crawlerNotInScope()
StateVertex	crawlIndex()	This method calls the index state.
private void	crawlThroughActions()
private void	crawlThroughActionsNew()	Crawl through the actions of the current state.
private void	crawlThroughActionsOld()
private boolean	domChanged(Eventable eventable, StateVertex newState)
void	execute(StateVertex crawlTask)
private boolean	fireEvent(Eventable eventable, boolean resolveXpath)	Try to fire a given event on the Browser.
boolean	fireEventWithInputs(Eventable event)
boolean	fireEventWithInputs(Eventable event, List<FormInput> available)
private ArrayList<Eventable>	follow(CrawlPath path, StateVertex targetState)
private ArrayList<Eventable>	followNew(CrawlPath path, StateVertex targetState)
private ArrayList<Eventable>	followOld(CrawlPath path, StateVertex targetState)
private long	getBackTrackingEventableId()
CrawlerContext	getContext()
CrawlRules	getCrawlRules()
private long	getEventableId()
private List<FormInput>	getInputElements(Eventable eventable)
private List<FormInput>	getNextPair(Map<FormInput,Boolean> tries)
List<StateVertex>	getOnUrlSet()
private Eventable	getResetEquivalent()
private void	goBackOneState()
private List<FormInput>	handleInputElements(Eventable eventable)	Enters the form data.
private List<FormInput>	handleInputs_pairwise(Eventable event, List<FormInput> available)
private boolean	inspectNewDom(Eventable event, StateVertex newState)
(package private) boolean	inspectNewState(Eventable event)
private void	interruptThread()
boolean	isValid(Eventable eventable)
private List<FormInput>	orderFormInputs(List<FormInput> formInputs)	Computes visual order
private void	parseCurrentPageForCandidateElements()
private long	parseWaitTimeOrReturnDefault(Matcher m)
static String	printCrawlPath(List<Eventable> path, boolean print)
private boolean	reachFromHome(StateVertex crawlTask)
void	reset(int nextTarget)	Resets crawljax by navigating to home url
private Eventable	resolveByXpath(Eventable eventable, Eventable eventToFire)
private void	setBTStatus(boolean success, int reachedNd)
(package private) void	setStateMachine(StateMachine sm)
private com.google.common.collect.ImmutableList<Eventable>	shortestPathTo(StateVertex crawlTask)
private com.google.common.collect.ImmutableList<Eventable>	shortestPathTo(StateVertex crawlTask, StateVertex onURL)
private void	tryToFireEvent(StateVertex targetState, StateVertex curState, Eventable clickable)
private boolean	visitAnchorHrefIfPossible(Eventable eventable)
private void	waitForRefreshTagIfAny(Eventable eventable)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

private static final org.slf4j.Logger LOG

DUPLICATE_EVENT_SEED

private static final int DUPLICATE_EVENT_SEED

See Also:

Constant Field Values

crawlDepth

private final AtomicInteger crawlDepth

maxDepth

private final int maxDepth

browser

private final EmbeddedBrowser browser

context

private final CrawlerContext context

stateComparator

private final StateComparator stateComparator

url

private final URI url

basicAuthUrl

private final URI basicAuthUrl

crawlScope

private final CrawlScope crawlScope

plugins

private final Plugins plugins

formHandler

private final FormHandler formHandler

crawlRules

private final CrawlRules crawlRules

waitConditionChecker

private final WaitConditionChecker waitConditionChecker

candidateExtractor

private final CandidateElementExtractor candidateExtractor

candidateActionCache

private final UnfiredFragmentCandidates candidateActionCache

graphProvider

private final javax.inject.Provider<InMemoryStateFlowGraph> graphProvider

vertexFactory

private final StateVertexFactory vertexFactory

fragmentManager

private final FragmentManager fragmentManager

crawlpath

private CrawlPath crawlpath

stateMachine

private StateMachine stateMachine

nextEventableId

private long nextEventableId

avoidUnrelatedBacktracking

private boolean avoidUnrelatedBacktracking

UseEquivalentReset

private boolean UseEquivalentReset

avoidDifferentBacktracking

private boolean avoidDifferentBacktracking

pairwiseFormHandling

private final boolean pairwiseFormHandling

See Also:

Constant Field Values

BACKTRACKING_SEED

private static final long BACKTRACKING_SEED

See Also:

Constant Field Values

nextBackTrackingId

private long nextBackTrackingId

Constructor Detail

Crawler

@Inject
Crawler(CrawlerContext context,
        CrawljaxConfiguration config,
        StateComparator stateComparator,
        UnfiredFragmentCandidates candidateActionCache,
        CoreModule.FormHandlerFactory formHandlerFactory,
        CoreModule.TrainingFormHandlerFactory trainingFormHandlerFactory,
        WaitConditionChecker waitConditionChecker,
        CoreModule.CandidateElementExtractorFactory elementExtractor,
        javax.inject.Provider<InMemoryStateFlowGraph> graphProvider,
        Plugins plugins,
        StateVertexFactory vertexFactory)

Method Detail

setStateMachine

void setStateMachine(StateMachine sm)

printCrawlPath

public static String printCrawlPath(List<Eventable> path,
                                    boolean print)

addPathInfo

private static void addPathInfo(CrawlPath path,
                                StringBuilder builder)

close

public void close()

Close the browser.

getResetEquivalent

private Eventable getResetEquivalent()

reset

public void reset(int nextTarget)

Resets crawljax by navigating to home url

Parameters:

nextTarget -

checkOnURLState

private void checkOnURLState(StateVertex previousState)

execute

public void execute(StateVertex crawlTask)

Parameters:

crawlTask - The StateVertex this Crawler should visit to crawl.

setBTStatus

private void setBTStatus(boolean success,
                         int reachedNd)

reachFromHome

private boolean reachFromHome(StateVertex crawlTask)

shortestPathTo

private com.google.common.collect.ImmutableList<Eventable> shortestPathTo(StateVertex crawlTask,
                                                                          StateVertex onURL)

shortestPathTo

private com.google.common.collect.ImmutableList<Eventable> shortestPathTo(StateVertex crawlTask)

follow

private ArrayList<Eventable> follow(CrawlPath path,
                                    StateVertex targetState)

followOld

private ArrayList<Eventable> followOld(CrawlPath path,
                                       StateVertex targetState)
                                throws CrawljaxException

Throws:

CrawljaxException

followNew

private ArrayList<Eventable> followNew(CrawlPath path,
                                       StateVertex targetState)
                                throws CrawljaxException

Throws:

CrawljaxException

checkCrawlConditions

private void checkCrawlConditions(StateVertex targetState)

changeState

private StateVertex changeState(StateVertex targetState,
                                Eventable clickable)

tryToFireEvent

private void tryToFireEvent(StateVertex targetState,
                            StateVertex curState,
                            Eventable clickable)

handleInputElements

private List<FormInput> handleInputElements(Eventable eventable)

Enters the form data. First, the related input elements (if any) to the eventable are filled in and then it tries to fill in the remaining input elements.

Parameters:

eventable - the eventable element.

Returns:

getInputElements

private List<FormInput> getInputElements(Eventable eventable)

orderFormInputs

private List<FormInput> orderFormInputs(List<FormInput> formInputs)

Computes visual order

Parameters:

formInputs -

Returns:

isValid

public boolean isValid(Eventable eventable)

getNextPair

private List<FormInput> getNextPair(Map<FormInput,Boolean> tries)

handleInputs_pairwise

private List<FormInput> handleInputs_pairwise(Eventable event,
                                              List<FormInput> available)

fireEventWithInputs

public boolean fireEventWithInputs(Eventable event,
                                   List<FormInput> available)

fireEventWithInputs

public boolean fireEventWithInputs(Eventable event)

fireEvent

private boolean fireEvent(Eventable eventable,
                          boolean resolveXpath)

Try to fire a given event on the Browser.

Parameters:

eventable - the eventable to fire

Returns:

true iff the event is fired

resolveByXpath

private Eventable resolveByXpath(Eventable eventable,
                                 Eventable eventToFire)

visitAnchorHrefIfPossible

private boolean visitAnchorHrefIfPossible(Eventable eventable)

crawlThroughActionsOld

private void crawlThroughActionsOld()

crawlThroughActions

private void crawlThroughActions()

crawlThroughActionsNew

private void crawlThroughActionsNew()

Crawl through the actions of the current state. The browser keeps firing CandidateCrawlActions stored in the state until the DOM changes. When it does, it checks if the new dom is a clone or a new state. In continues crawling in that new or clone state. If the browser leaves the current domain, the crawler tries to get back to the previous state.

The methods stops when Thread.interrupted()

getBackTrackingEventableId

private long getBackTrackingEventableId()

getEventableId

private long getEventableId()

interruptThread

private void interruptThread()

inspectNewState

boolean inspectNewState(Eventable event)

domChanged

private boolean domChanged(Eventable eventable,
                           StateVertex newState)

inspectNewDom

private boolean inspectNewDom(Eventable event,
                              StateVertex newState)

parseCurrentPageForCandidateElements

private void parseCurrentPageForCandidateElements()

waitForRefreshTagIfAny

private void waitForRefreshTagIfAny(Eventable eventable)

crawlerNotInScope

private boolean crawlerNotInScope()

parseWaitTimeOrReturnDefault

private long parseWaitTimeOrReturnDefault(Matcher m)

goBackOneState

private void goBackOneState()

crawlIndex

public StateVertex crawlIndex()

This method calls the index state. It should be called once per crawl in order to setup the crawl.

Returns:

The initial state.

getContext

public CrawlerContext getContext()

getCrawlRules

public CrawlRules getCrawlRules()

getOnUrlSet

public List<StateVertex> getOnUrlSet()