public final class ScrapingSession extends java.lang.Object implements ScriptInstancesContainer, Deleteable, RunningScrapingSession, java.lang.Comparable, java.awt.datatransfer.Transferable, Folderable
Modifier and Type | Class and Description |
---|---|
(package private) class |
ScrapingSession.BreakpointFrameDisposer |
(package private) class |
ScrapingSession.BreakpointFrameLauncher |
Modifier and Type | Field and Description |
---|---|
(package private) static java.awt.datatransfer.DataFlavor[] |
flavors |
static java.awt.datatransfer.DataFlavor |
INFO_FLAVOR
Drag 'n drop stuff.
|
static int |
INT_ACCEPT |
static int |
INT_APACHE_HTTPCLIENT |
static int |
INT_ASYNC_HTTPCLIENT_2 |
static int |
INT_BEST_MATCH |
static int |
INT_CURL_HTTPCLIENT |
static int |
INT_HTTP_CLIENT_HTTPCLIENT |
static int |
INT_HTTP_CLIENT_IE |
static int |
INT_HTTP_CLIENT_MOZILLA |
static int |
INT_NING_ASYNC_HTTPCLIENT |
static int |
INT_REJECT |
static int |
INT_STRICT_MODE |
static org.apache.log4j.Logger |
log
Used for logging.
|
static java.lang.String |
STRING_ACCEPT |
static java.lang.String |
STRING_ACCORDING_TO_COOKIE_SPEC
Cookie policy constants.
|
static java.lang.String |
STRING_APACHE_HTTPCLIENT
Constants used to determine the client to use for the session.
|
static java.lang.String |
STRING_ASYNC_HTTPCLIENT_2 |
static java.lang.String |
STRING_BEST_MATCH |
static java.lang.String |
STRING_COMPATIBILITY |
static java.lang.String |
STRING_CURL_HTTPCLIENT |
static java.lang.String |
STRING_HTTP_CLIENT_HTTPCLIENT |
static java.lang.String |
STRING_HTTP_CLIENT_IE |
static java.lang.String |
STRING_HTTP_CLIENT_MOZILLA |
static java.lang.String |
STRING_NETSCAPE_DRAFT |
static java.lang.String |
STRING_NING_ASYNC_HTTPCLIENT |
static java.lang.String |
STRING_REJECT |
static java.lang.String |
STRING_STRICT_MODE |
Constructor and Description |
---|
ScrapingSession()
Needed by castor.
|
ScrapingSession(int scrapingSessionID,
java.lang.String name,
int cookieHandling,
int cookiePolicy,
int httpClientType,
int maxHTTPRequests,
java.lang.String externalProxyUsername,
java.lang.String externalProxyPassword,
java.lang.String externalProxyHost,
java.lang.String externalProxyPort,
java.lang.String externalNTProxyUsername,
java.lang.String externalNTProxyPassword,
java.lang.String externalNTProxyHost,
java.lang.String externalNTProxyPort,
boolean useStrictMode,
int folderID,
java.lang.String notes,
boolean anonymize,
boolean terminateProxiesOnCompletion,
int numberOfRequiredProxies,
int originatorEdition,
int loggingLevel,
java.lang.String characterSet,
java.lang.String keyStoreFilePath,
java.lang.String keyStorePassword,
boolean useOnlySSLv3,
java.lang.String createdByVersion,
long dateUpdated)
Generates a new ScrapingSession with all of its basic information.
|
ScrapingSession(ScrapingSession scrapingSession)
Generates a new scraping session as a copy of the given one.
|
ScrapingSession(java.lang.String name)
Generates a new ScrapingSession, given a string.
|
Modifier and Type | Method and Description |
---|---|
void |
addAllExecuteScriptScriptsToExport()
Adds all execute script scripts that need to be exported by examining each script invoked from within this scraping session for invocations of
session.executeScript.
|
void |
addAutoProgressBar(java.lang.String name,
java.util.Collection<java.lang.String> values)
Creates an automatic progress bar and adds it to the progress bars.
|
void |
addAutoProgressBar(java.lang.String name,
DataSet values,
java.lang.String key)
Creates an automatic progress bar and adds it to the progress bars.
|
void |
addAutoProgressBar(java.lang.String name,
java.lang.String... values)
Creates an automatic progress bar and adds it to the progress bars.
|
void |
addAutoProgressBar(java.lang.String name,
java.lang.String[][] values)
Creates an automatic progress bar and adds it to the progress bars.
|
void |
addAutoProgressBar(java.lang.String name,
java.lang.String[][] values,
int keyIndex)
Creates an automatic progress bar and adds it to the progress bars.
|
protected void |
addChildScrapingSession(ScrapingSession childScrapingSession)
Adds a child ScrapingSession object.
|
<T extends AbstractEventData> |
addEventCallback(EventFireTime<T> eventTime,
EventHandler<T> callback)
Add a runnable that will be executed at the given time.
|
<T extends AbstractEventData> |
addEventCallbackWithPriority(EventFireTime<T> eventTime,
EventHandler<T> callback,
int priority)
Add a runnable that will be executed at the given time and priority.
|
void |
addExecuteScriptScriptToExport(Script script)
Adds a script to be exported that gets invoked via session.executeScript.
|
void |
addMonitoredPostfix(java.lang.String postfix)
Watches for all session variables whose keys end with the postfix specified, and will output their values when monitored variables are logged.
|
void |
addMonitoredPrefix(java.lang.String prefix)
Watches for all session variables whose keys begin with the prefix specified, and will output their values when monitored variables are logged.
|
java.lang.Object |
addMonitoredValue(java.lang.String name,
java.lang.Object value)
Adds a specific name and value to be logged with the web messages methods or logMonitoredValues method
|
void |
addMonitoredVariable(java.lang.String key)
Watches the value of a session variable, and will output it each time monitored variables are output
|
ProgressBar |
addProgressBar(java.lang.String title)
Adds a new progress bar with the given title
|
ProgressBar |
addProgressBar(java.lang.String title,
double total)
Adds a new progress bar with the given title and total
|
ProgressBar |
addProgressBar(java.lang.String title,
java.lang.String total)
Adds a new progress bar with the given title and total
|
void |
addRandomDelayBetweenRequests(long minMs,
long maxMs)
Adds a random delay between requests, ranging from the min to max number of milliseconds.
|
void |
addScrapeableFile(ScrapeableFile scrapeableFile)
Inserts a ScrapeableFile into the session.
|
void |
addSessionVariableMonitor(java.lang.String monitoredVariable)
Deprecated.
|
void |
addToNumDuplicateRecordsScraped(int numRecordsScraped)
Adds to the number of duplicate records scraped.
|
void |
addToNumErrorRecordsScraped(int numRecordsScraped)
Adds to the number of error records scraped.
|
void |
addToNumNewRecordsScraped(int numRecordsScraped)
Adds to the number of new records scraped.
|
void |
addToNumRecordsScraped(int numRecordsScraped)
Adds to the number of records scraped.
|
void |
addToNumRecordsScraped(java.lang.Object numRecordsScraped)
Adds to the number of records scraped.
|
void |
addToNumUpdatedRecordsScraped(int numRecordsScraped)
Adds to the number of updated records scraped.
|
void |
addToVariable(java.lang.String variable,
int value)
Adds a value to a session variable.
|
void |
addXMLFeedItem(java.lang.String title,
java.lang.String link,
java.lang.String description,
java.util.Date publishedDate)
Adds an XML feed entry to the session.
|
void |
appendErrorMessage(java.lang.String errorMessage)
Adds a message to the existing error message(s).
|
void |
appendStatusMessage(java.lang.String statusMessage)
Adds a message to the existing status message(s).
|
void |
breakpoint()
Performs a breakpoint.
|
void |
breakpoint(boolean logVariablesInNonWorkbench)
Pauses the scrape and opens the breakpoint window.
|
void |
breakpoint(java.lang.Object message)
Pauses the scrape and opens the breakpoint window.
|
void |
breakpoint(java.lang.Object message,
boolean logVariablesInNonWorkbench)
Pauses the scrape and opens the breakpoint window.
|
void |
breakpoint(ScriptContext scriptContext)
Performs a breakpoint.
|
void |
breakpointIfNeeded()
Breakpoints, if needed.
|
void |
bumpScrapeableFileSequences(int minimumSequence,
int maximumSequence,
boolean bumpUp)
Bumps the sequence up or down by one for any
ScrapeableFile whose sequence falls in between the given sequences. |
java.io.File |
cacheBytes(java.lang.String outputFilenameAndPath,
byte[] bytes)
Adds a file to the cache from bytes.
|
java.io.File |
cacheFile(java.lang.String outputFilenameAndPath,
java.io.File fileToCache)
Adds a file to the cache.
|
java.io.File |
cacheScrapeableFile(ScrapeableFile scrapeableFile)
Caches the HTML and headers of the scrapeable file
|
java.io.File |
cacheText(java.lang.String name,
java.lang.String content)
Adds text to the cache.
|
java.io.File |
cacheText(java.lang.String name,
java.lang.String content,
java.lang.String encoding)
Adds text to the cache.
|
void |
clearAllSessionVariables()
Clears all session variables.
|
<T extends AbstractEventData> |
clearCallbacksForTime(EventFireTime<T> eventTime)
Clears the callbacks for the specified time
|
void |
clearCookies()
Clears all cookies.
|
void |
clearCustomScriptVariables()
Clears the custom script variables for this scraping session
|
void |
clearLastRequestsAndResponses()
Clears the last request and response for each of the constituent scrapeable files.
|
void |
clearProxySettings() |
void |
clearVariables(java.util.Collection<java.lang.Object> variables)
Clears the value of all session variables that that match the values in the collection
|
void |
clearVariables(java.util.Map<java.lang.Object,java.lang.Object> variables)
Clears the value of all session variables that that match the keys in the Map
|
int |
compareTo(java.lang.Object object)
Required by the
Comparable interface. |
void |
convertHTMLEntitiesInVariable(java.lang.String variable)
Replaces HTML entities in a variable with their non-entity equivalents.
|
void |
currentProxyServerIsBad()
Removes the current
ProxyServer from the pool. |
void |
debug(java.lang.Object message)
Sends an app event off to the view to be logged as DEBUG.
|
void |
debug(java.lang.Object message,
java.lang.Throwable throwable)
Sends an app event off to the view to be logged as DEBUG.
|
void |
delete()
Deletes the session from the database.
|
void |
determineUsesNavigationActions()
Determines and sets whether or not this scraping session uses navigation actions.
|
void |
doExtendedReset(boolean nullScrapingSessionNotifiables)
Resets any remaining objects, which external entities querying the scraping session for data may otherwise rely on.
|
boolean |
doLazyScrape()
Indicates whether or not the session should do a "lazy" scrape.
|
boolean |
downloadFile(java.lang.String urlString,
java.lang.String fileName)
A utility method for downloading a resource and writing it to a file.
|
boolean |
downloadFile(java.lang.String urlString,
java.lang.String fileName,
int maxNumAttempts)
A utility method for downloading a resource and writing it to a file.
|
boolean |
downloadFile(java.lang.String urlString,
java.lang.String fileName,
int maxNumAttempts,
boolean doLazy)
A utility method for downloading a resource and writing it to a file.
|
void |
dumpToLog()
Dumps the session to the log.
|
void |
enableCaching(java.lang.String description,
boolean saveLogs,
boolean zipCachedFiles)
Enables caching for this scrape.
|
java.io.File |
endCaching()
Ends the cache for the scrape.
|
void |
endCachingAtScrapeClose()
Ends the cache for the scrape.
|
void |
ensureFullyLoaded()
Ensures that the scraping session is completely loaded.
|
void |
ensureScrapeableFilesHaveScrapingSessionReference()
Ensures that all constituent scrapeable file objects have a reference to this scraping session.
|
boolean |
equals(java.lang.Object object)
Overriding equals...
|
void |
error(java.lang.Object message)
Sends an app event off to the view to be logged as ERROR.
|
void |
error(java.lang.Object message,
java.lang.Throwable throwable)
Sends an app event off to the view to be logged as ERROR.
|
void |
executeScript(java.lang.String scriptName)
Executes the named script.
|
void |
executeScriptWithContext(java.lang.String scriptName)
Executes the named script, but preserves the current context (dataRecord, scrapeableFile, etc...)
|
void |
export(java.io.OutputStream outputStream)
Exports the scraping session to the given stream.
|
protected void |
finalize() |
void |
flagAsLoaded()
Flags the scraping session as having been loaded.
|
void |
generateScrapeableSession()
Generates a scrapeable session that will correspond to a running instance of this scraping session.
|
java.util.Map<java.lang.String,java.lang.Object> |
getAllInternalVariables()
Returns a duplicate of the session internal variables.
|
protected java.lang.String |
getAllSessionVariablesAsURLEncodedString()
Gets all session variables as a URL-encoded string.
|
java.util.Map<java.lang.String,java.lang.Object> |
getAllVariables()
Returns a duplicate of the session variables.
|
boolean |
getAnonymize()
Indicates whether or not the scrape should be anonymized
|
boolean |
getCachingEnabled()
Returns whether or not the cache is enabled for the scrape
|
java.lang.String |
getCharacterSet()
Gets the current character set being used.
|
static java.lang.String |
getCharacterSetFromScrapingSessionExportFile(java.io.File file)
Determines the character set of the scraping session by examining the contents of its export file.
|
static java.lang.String |
getCharacterSetFromScrapingSessionExportFile(java.io.File file,
java.lang.String charSet)
Determines the character set of the scraping session by examining the contents of its export file.
|
int |
getConnectionTimeout()
Gets the connection timeout to use.
|
java.util.List<ScrapingCookie> |
getCookieList()
Returns a list of cookies for the current session
|
int |
getCookiePolicy()
Gets the cookie policy option.
|
java.lang.String |
getCookiePolicyAsHttpClientString()
Gets the cookie policy as a
String that is friendly to HttpClient 4. |
org.apache.commons.httpclient.Cookie[] |
getCookies()
Deprecated.
Use the
getCookieList() method instead. |
java.lang.String |
getCreatedByVersion()
Gets the version of ss that originally created the scraping session.
|
ProxyServer |
getCurrentProxyServerFromPool()
Gets a reference to the current
ProxyServer being used from the pool. |
ScriptContext |
getCurrentScriptContext()
Gets the current
ScriptContext . |
java.lang.String[] |
getCurrentStack()
Returns the current stack of scripts
|
java.lang.Object |
getCustomScriptVariable(java.lang.String name)
Returns the current value for the given custom script variable
|
java.util.Map<java.lang.String,java.lang.Object> |
getCustomScriptVariables()
Returns a map of the current script variables.
|
java.lang.Object[] |
getDataToSendToClientAsArrayAndFlush()
Gets the data to be sent to the client as a raw array of
HashMap objects, then flushes out all of the data from the array list. |
java.lang.String |
getDateExported()
Gets the date the scraping session was exported.
|
long |
getDateUpdated()
Gets the last time the scraping session was updated.
|
boolean |
getDebugMode()
Checks to see if this is currently set to run in debug mode
|
RetryPolicy |
getDefaultRetryPolicy()
Gets the default retry policy to be used by each scrapeable file when one wasn't set for it.
|
java.lang.String |
getDefaultUserAgent()
Gets the default user agent.
|
long |
getElapsedRunningTime()
Gets the number of milliseconds that have passed since the scraping session started.
|
java.lang.String |
getErrorMessage()
Gets the error message that should be persisted to the database once the scraping session ends.
|
EventEngine |
getEventEngine()
Returns the event engine used to add custom event callbacks
|
EventEngine |
getEventEngineInternal()
Returns the event engine used to add custom event callbacks.
|
java.util.HashSet |
getExecuteScriptScriptsToExport()
Gets the set that holds scripts that need to be exported that are called via session.executeScript.
|
java.lang.String |
getExternalNTProxyDomain()
Gets the external NT proxy port.
|
java.lang.String |
getExternalNTProxyHost()
Gets the external NT proxy host.
|
java.lang.String |
getExternalNTProxyPassword()
Gets the external NT proxy password.
|
java.lang.String |
getExternalNTProxyUsername()
Gets the NT external proxy username.
|
java.lang.String |
getExternalProxyHost()
Gets the external proxy host.
|
java.lang.String |
getExternalProxyPassword()
Gets the external proxy password.
|
java.lang.String |
getExternalProxyPort()
Gets the external proxy port.
|
java.lang.String |
getExternalProxyUsername()
Gets the external proxy username.
|
boolean |
getFatalErrorOccurred()
Gets the flag used to indicate whether or not a fatal error occurred while running this scraping session.
|
Folder |
getFolder()
Gets the folder containing this object.
|
boolean |
getHasBeenQueued()
Indicates whether or not the scraping session has been queued to be run.
|
ScrapingHttpClient |
getHttpClient()
Gets the HTTP client to be used in scraping.
|
int |
getHTTPClientType()
Gets the HTTP client to be used throughout the session.
|
int |
getID()
Gets the ID of the scraping session.
|
java.lang.String |
getIdentifier() |
int |
getImageHeight(java.lang.String imagePath)
Gets the height of an image.
|
int |
getImageWidth(java.lang.String imagePath)
Gets the width of an image.
|
<T> T |
getInternalVariable(java.lang.String key)
Gets the value of an internal variable.
|
java.lang.String |
getKeyStoreFilePath()
Gets the path to the keystore file.
|
java.lang.String |
getKeyStorePassword()
Gets the password to be used for the key store file.
|
java.lang.String |
getLastDeterminedCharacterSet()
Gets the last character set determined from a server.
|
java.util.Date |
getLastImportTime()
In workbench mode returns the current time, otherwise returns the last time a scrape with this name was imported (or if it hasn't been imported since the
instance was started will return the instance start time)
|
ProgressBar |
getLastProgressBar()
Returns the last progress bar
|
ScrapeableFile |
getLatestActiveScrapeableFile()
Gets the scrapeable file most recently added to the active scrapeable files.
|
java.lang.String |
getLogFileName()
Gets the name of the log file corresponding to this scraping session.
|
int |
getLoggingLevel()
Gets the logging level to use when running the scraping session.
|
int |
getMaxConcurrentFileDownloads()
Gets the maximum number of concurrent file downloads to a allow.
|
int |
getMaxHTTPRequests()
Gets the maximum number of attempts that should be made for each HTTP request.
|
int |
getMaxScriptsOnStack()
Gets the maximum number of script context objects that can be pushed on to the stack.
|
int |
getMemoryUsage()
Gets the average memory usage.
|
java.lang.String |
getName()
Gets the name.
|
java.lang.String |
getNameUpperCase()
Gets the name in upper case (used for comparing with other scraping sessions).
|
boolean |
getNeedsToBeSaved()
Indicates whether or not the object needs to be saved.
|
java.lang.String |
getNextDefaultScrapeableFileName()
Gets the name of a new scrapeable file, ensuring it isn't a duplicate.
|
javax.swing.tree.DefaultMutableTreeNode |
getNode()
Gets the node representing this session.
|
ScrapeableFile |
getNonSequencedScrapeableFileBySequence(int sequence)
Gets a scrapeable file from the session that doesn't have a sequence, but is found by cycling through the scrapeable files alphabetically.
|
java.lang.String |
getNotes()
Gets the notes.
|
boolean |
getNotifyNotifiableOfCompletion()
Indicates whether or not the notifiable object should be notified once the scraping session completes.
|
int |
getNumActiveScrapeableFiles()
Gets the number of active scrapeable files.
|
int |
getNumberOfRequiredProxies()
The number of proxies required to anonymize this scraping session
|
int |
getNumDuplicateRecordsScraped()
Gets the number of duplicate records scraped for the primary data set corresponding to this scraping session.
|
int |
getNumErrorRecordsScraped()
Gets the number of error records scraped for the primary data set corresponding to this scraping session.
|
int |
getNumNewRecordsScraped()
Gets the number of new records scraped for the primary data set corresponding to this scraping session.
|
int |
getNumProgressBars()
Returns the current number of progress bars
|
int |
getNumRecordsScraped()
Gets the number of records scraped for the primary data set corresponding to this scraping session.
|
int |
getNumScrapeableFiles()
Returns the number of scrapeable files currently held by the session.
|
int |
getNumScrapeableFilesWithSequence()
Returns the number of scrapeable files currently held by the session that use a valid sequence.
|
int |
getNumScriptsOnStack()
Gets the number of
ScriptContext objects on the stack. |
int |
getNumUpdatedRecordsScraped()
Gets the number of duplicate records scraped for the primary data set corresponding to this scraping session.
|
int |
getOriginatorEdition()
Indicates the edition of screen-scraper in which the scraping session was created.
|
ScrapingSession |
getParentScrapingSession()
If this scraping session was invoked as a RunnableScrapingSession this gets its parent.
|
ProgressBar |
getProgressBar(int index)
Returns the progress bar at the given index, with 0 being the highest level bar, and lower-level bars being sequentially larger in index
|
ProgressBar |
getProgressBar(java.lang.String title)
Returns the first progress bar with the given title, or null if none is found.
|
ProxyServerPool |
getProxyServerPool()
Gets the current
ProxyServerPool object, used to cycle through a pool of proxies. |
boolean |
getRandomizeUserAgent()
Indicates whether or not the user agent should be randomized.
|
int |
getRelativeRecordsScrapedThreshold()
Gets the records scraped threshold to use for the scraping session.
|
int |
getRelativeTimeThreshold()
Gets the time threshold to use for the scraping session.
|
boolean |
getRetainNonTidiedHTML()
Gets whether or not the tidied HTML should be retained for all of the scrapeable files in this session.
|
ScrapeableFile |
getScrapeableFile(int sequence)
Gets a scrapeable file from the session.
|
ScrapeableFile |
getScrapeableFile(java.lang.String name)
Gets a scrapeable file from the session.
|
ScrapeableFile |
getScrapeableFileByID(int scrapeableFileID)
Gets a scrapeable file from the session by its ID.
|
ScrapeableFile |
getScrapeableFileCaseInsensitive(java.lang.String name)
Gets a scrapeable file from the session, without regard to case.
|
java.util.HashSet |
getScrapeableFiles()
Gets all
ScrapeableFile s associated with this scraping session. |
java.util.Iterator |
getScrapeableFilesIterator()
An
Iterator used to cycle through the ScrapeableFile objects held by this object. |
ScrapeableSession |
getScrapeableSession()
Gets the scrapeable session that corresponds to a running instance of this scraping session.
|
int |
getScrapeableSessionID()
When running, this will get the scrapeable session ID.
|
FileCacher |
getScrapeCache()
Returns the scrape cacher object used to cache data about the scrape
|
ScrapeStateLogger |
getScrapeLogger()
Gets the ScrapeStateLogger that is currently holding monitored variable names, etc...
|
ScrapingSessionNotifiable |
getScrapingSessionNotifiable()
Gets the object the scraping session will use to notify its progress of.
|
ScrapingSessionState |
getScrapingSessionState()
Gets the session state for scripts as scrapeable files are being processed.
|
ScriptInstances |
getScriptInstances()
Returns a
ScriptInstances object containing all script instances associated with this scrapeable file. |
boolean |
getShouldBreakpoint()
Gets whether or not the session should breakpoint at the next opportunity
|
long |
getStartTime()
Gets the starting time in milliseconds.
|
java.lang.String |
getStatusMessage()
Gets the status message that should be persisted to the database once the scraping session ends.
|
StopScrapingCallType |
getStopScrapingCallType()
Returns the call type that was issued to initially stop the scraping session
|
java.lang.String |
getStopScrapingExplanation()
Returns a string explaining the reason why the scrape was stopped initially.
|
boolean |
getStopScrapingOnExtractorPatternTimeout()
Gets whether or not scraping should stop in the event of an extractor pattern timeout
|
boolean |
getStopScrapingOnMaxRequestAttemptsReached()
Gets whether or not scraping should stop in the event of max request attempts reached.
|
boolean |
getStopScrapingOnScriptError()
Gets whether or not scraping should stop in the event of a script error.
|
boolean |
getTerminateProxiesOnCompletion()
Indicates whether or not proxies should be terminated on completion
|
int |
getTimeout()
Gets the timeout, in minutes, for the scraping session.
|
java.util.TimeZone |
getTimeZone()
Gets the current time zone of the Scraping Session
|
java.lang.Object |
getTransferData(java.awt.datatransfer.DataFlavor df) |
java.awt.datatransfer.DataFlavor[] |
getTransferDataFlavors() |
java.net.URL |
getURLToUseAsReferer()
Gets the URL that should be used for the referer HTTP header for scrapeable files subsequent to this one.
|
FileCacher |
getUsableScrapeCache()
Returns the scrape cacher object used to cache data about the scrape, or the parent scrapes if this scrape has a parent but not a cacher
|
boolean |
getUseOnlySSLv3()
Gets whether or not only SSLv3 should be used when connecting to remote servers.
|
boolean |
getUseProxyFromPool()
Indicates whether or not a proxy should be used for the current request(s).
|
java.lang.String |
getUserAgent()
Gets the user-agent to be used in HTTP requests.
|
boolean |
getUseServerCharacterSet()
Gets whether the character set indicated by the server should be used.
|
boolean |
getUsesNavigationActions()
Indicates whether or not any scrapeable files held by this scraping session use navigation actions.
|
boolean |
getUseStrictMode()
Indicates whether or not strict mode should be used.
|
java.lang.Object |
getv(java.lang.String variable)
Alias to getVariable.
|
java.lang.Object |
getVariable(java.lang.String variable)
Gets the value of a session variable.
|
com.gargoylesoftware.htmlunit.WebClient |
getWebClient()
Gets the HTMLUnit web client used in navigating.
|
void |
handleScriptError(ScriptException scriptException,
ScriptContext scriptContext)
Handles reporting and responding to a script error.
|
int |
hashCode() |
void |
info(java.lang.Object message)
Sends an app event off to the view to be logged as INFO.
|
void |
info(java.lang.Object message,
java.lang.Throwable throwable)
Sends an app event off to the view to be logged as INFO.
|
boolean |
isCompleted()
Indicates whether or not the scraper has finished scraping.
|
boolean |
isCreatedByVersionAtLeast(java.lang.String compareToVersion)
Indicates whether or not the version that created this scraping session is greater than or equal to the given version.
|
boolean |
isDataFlavorSupported(java.awt.datatransfer.DataFlavor df)
************************************************** Transferable methods.
|
boolean |
isRunning()
Indicates whether or not the session is currently running.
|
boolean |
isRunningFromCommandLine()
Returns whether or not we are currently running from the command line.
|
boolean |
isRunningInServer()
Returns whether or not we are currently running in server mode.
|
boolean |
isRunningInWorkbench()
Returns whether or not we are currently running in the workbench.
|
void |
loadStateFromString(java.lang.String stateXML)
Loads the scrape state from an xml string, but doesn't clear whatever was there.
|
void |
loadVariables(java.lang.String fileToReadFrom)
Reads in and loads the session variables that were previously saved to a file.
|
void |
log(int message)
Sends an app event off to the view to be logged.
|
void |
log(java.lang.Object message)
Sends an app event off to the view to be logged.
|
void |
logCurrentDateAndTime()
Logs the current date and time.
|
void |
logCurrentTime()
Logs the current time.
|
void |
logDataRecord(DataRecord record)
Logs all the values in a Data Record to the log, with one line per value.
|
void |
logDataRecord(DataRecord record,
int logLevel)
Logs all the values in a Data Record to the log, with one line per value.
|
void |
logDataRecordDebug(DataRecord record)
Logs a DataRecord at the Debug level
|
void |
logDataRecordError(DataRecord record)
Logs a DataRecord at the Error level
|
void |
logDataRecordInfo(DataRecord record)
Logs a DataRecord at the Info level
|
void |
logDataRecordWarn(DataRecord record)
Logs a DataRecord at the Warn level
|
void |
logDebug(java.lang.Object message)
Sends an app event off to the view to be logged as DEBUG.
|
void |
logDebug(java.lang.String message)
Sends an app event off to the view to be logged as DEBUG.
|
void |
logElapsedRunningTime()
Logs the number of days, hours, minutes, and seconds from the time the scraping session started.
|
void |
logError(java.lang.Object message)
Sends an app event off to the view to be logged as ERROR.
|
void |
logError(java.lang.Object message,
java.lang.Throwable t)
Sends an app event off to the view to be logged as ERROR.
|
void |
logError(java.lang.String message)
Sends an app event off to the view to be logged as ERROR.
|
void |
logException(java.lang.Throwable exception)
Logs an Exception, with a full stack trace
|
void |
logInfo(java.lang.Object message)
Sends an app event off to the view to be logged as INFO.
|
void |
logInfo(java.lang.String message)
Sends an app event off to the view to be logged as INFO.
|
void |
logMemoryUsage()
Logs memory usage for variables and such.
|
void |
logMonitoredValues(java.lang.Object message)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logMonitoredValues(java.lang.Object message,
int logLevel)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logMonitoredValuesClose(java.lang.Object message)
Logs closing values to indicate the scrape is complete and what values were when everything finished.
|
void |
logMonitoredValuesDebug(java.lang.Object message)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logMonitoredValuesError(java.lang.Object message)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logMonitoredValuesInfo(java.lang.Object message)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logMonitoredValuesWarn(java.lang.Object message)
Logs the values of all the currently monitored variables, the progress of the scrape, if known, and puts the message at the top.
|
void |
logObjectByType(java.lang.Object object)
Logs the Object in a semi intelligent way.
|
void |
logObjectByType(java.lang.Object object,
int logLevel)
Logs the Object in a semi intelligent way.
|
void |
logObjectByTypeDebug(java.lang.Object object)
Logs the Object in a semi intelligent way.
|
void |
logObjectByTypeError(java.lang.Object object)
Logs the Object in a semi intelligent way.
|
void |
logObjectByTypeInfo(java.lang.Object object)
Logs the Object in a semi intelligent way.
|
void |
logObjectByTypeWarn(java.lang.Object object)
Logs the Object in a semi intelligent way.
|
void |
logScreenScraperInformation()
Logs useful information about the current instance of Screen-Scraper, as well as the Java VM and the General Utility version being used.
|
void |
logVariables()
Outputs all session variables to the log.
|
void |
logVariables(ScriptContext scriptContext)
Outputs all session variables to the log.
|
void |
logWarn(java.lang.Object message)
Sends an app event off to the view to be logged as WARN.
|
void |
logWarn(java.lang.String message)
Sends an app event off to the view to be logged as WARN.
|
static int |
mapCookiePolicyHumanReadableStringToInt(java.lang.String cookiePolicy)
Maps a cookie policy string to an int.
|
static java.lang.String |
mapCookiePolicyIntToHumanReadableString(int cookiePolicy)
Maps a cookie policy int to a human-readable string.
|
static java.lang.String |
mapHTTPClientIntToString(int httpClient)
Maps an HTTP client int to a human-readable string.
|
static int |
mapHTTPClientTypeStringToInt(java.lang.String httpClientType)
Maps an HTTP client string to an int.
|
void |
markStopTime()
Marks and sets the point at which the session should stop, if a timeout exists.
|
void |
moveToNewFolder(Folder folder)
Moves the object to a new folder.
|
void |
pause(long time)
Indicates that the scraping session should pause for the given number of milliseconds.
|
void |
popScriptContext()
Pops the top
ScriptContext object off the stack. |
void |
prepareAnonymization()
If the scrape is set to be anonymized this will initialize all the required settings and launch the required number of proxy servers.
|
protected void |
processPostScrapingScripts()
Processes any scripts after files are to be scraped.
|
protected void |
processPreScrapingScripts()
Processes any scripts before files are to be scraped.
|
void |
pushScriptContext(ScriptContext scriptContext)
Pushes a
ScriptContext object on to the stack. |
static boolean |
readerPointsToImportableScrapingSession(java.io.Reader reader)
Indicates whether or not the given reader points to an importable scraping session.
|
java.lang.String |
reformatDate(java.lang.String date,
java.lang.String dateFormatTo)
Reformats a date.
|
java.lang.String |
reformatDate(java.lang.String date,
java.lang.String dateFormatFrom,
java.lang.String dateFormatTo)
Reformats a date.
|
protected void |
removeChildScrapingSession(ScrapingSession childScrapingSession)
Removes a child ScrapingSession object.
|
<T extends AbstractEventData> |
removeEventCallback(EventFireTime<T> eventTime,
EventHandler<T> callback)
Remove a runnable that will be executed at the given time
|
void |
removeFromFolder()
Removes the object from its current folder.
|
void |
removeMonitoredPostfix(java.lang.String postfix)
Stops watching for a postfix in session variables
|
void |
removeMonitoredPrefix(java.lang.String prefix)
Stops watching for a prefix in session variables
|
java.lang.Object |
removeMonitoredValue(java.lang.String name)
Removes a specific name from the manually set values to be logged.
|
void |
removeMonitoredVariable(java.lang.String key)
Stops watching the specified variable
|
void |
removeProgressBar(ProgressBar progressBar)
Removes the specified progress bar
|
void |
removeScrapeableFile(int sequence)
Removes a scrapeable file from the session.
|
void |
removeScrapeableFile(java.lang.String name)
Removes a scrapeable file from the session.
|
void |
removeSessionVariablesInMap(java.util.Map<java.lang.Object,java.lang.Object> variables,
boolean ignoreLowerCaseKeys)
Clears the value of all session variables that that match the keys in the Map
|
void |
replaceScrapeableFile(ScrapeableFile scrapeableFile)
Replaces an existing scrapeable file with the given scrapeable file carrying the same name.
|
(package private) void |
resetCallbacks() |
void |
resetContainsNullableTokens()
Causes the containsNullableTokens value for each extractor pattern associated with each scrapeable file to be reset.
|
void |
resetInitialValues()
Resets a series of values to ensure that the scraping session starts fresh.
|
void |
resetNumRecordsScraped()
Resets the number of records scraped to 0.
|
boolean |
resetOnCompletion()
Indicates whether or not the session should be reset once it completes.
|
void |
resetReferers()
Resets the referer for all scrapeable files to null.
|
void |
resetScrapingSessionState()
Resets the scraping session state.
|
(package private) void |
resetThreadGate()
Resets the state of the thread gate so it is ready for a new scrape to run
|
void |
resizeImageFixHeight(java.lang.String originalFile,
java.lang.String thumbnailFile,
int newHeightSize,
boolean deleteOriginalFile)
Creates a thumbnail for an existing image.
|
void |
resizeImageFixWidth(java.lang.String originalFile,
java.lang.String thumbnailFile,
int newWidthSize,
boolean deleteOriginalFile)
Creates a thumbnail for an existing image.
|
void |
resizeImageFixWidthAndHeight(java.lang.String originalFile,
java.lang.String thumbnailFile,
int newWidthSize,
int newHeightSize,
boolean deleteOriginalFile)
Creates a thumbnail for an existing image.
|
void |
restoreHTTPParametersAfterScraping()
Indicates that HTTP parameters should be restored after the scraping session finishes scraping.
|
void |
save()
Saves the scraping session to the database.
|
java.lang.String |
saveStateToString(boolean saveCookies,
boolean saveVariables)
Saves the scrape state to an XML string
|
java.lang.String |
saveStateToString(boolean saveCookies,
java.util.Collection<java.lang.String> saveVariables)
Saves the scrape state to an XML string
|
void |
saveVariables(java.lang.String fileToSaveTo)
Writes out the current session variables to a file.
|
void |
scrape()
Required by the RunnableScrapingSession interface.
|
void |
scrapeFile(ScrapeableFile scrapeableFile)
Scrapes a file.
|
void |
scrapeFile(java.lang.String scrapeableFileName)
Scrapes a file.
|
void |
scrapeString(ScrapeableFile scrapeableFile,
java.lang.String content)
Scrapes a file using the given string for the last response instead of issuing a request
|
void |
scrapeString(java.lang.String scrapeableFileName,
java.lang.String content)
Calls a scrapeable file using the given contents as the response rather than issuing a request
|
void |
sendDataToClient(java.lang.String key,
java.lang.Object value)
Queues up an element of data to be sent to a client.
|
void |
sendMail(java.lang.String subject,
java.lang.String body,
java.lang.String recipients,
java.lang.String attachments,
java.lang.String headers)
Sends mail to a pre-specified SMTP server.
|
boolean |
sessionTimedOut()
Indicates whether or not the session timed out.
|
void |
setAnonymize(boolean anonymize)
Indicates whether or not the scrape should be anonymized
|
void |
setAutoCloseAfterScrapeEnds(java.io.Closeable closeable)
Sets the given closeable to be automatically closed once the session completes.
|
void |
setCharacterSet(java.lang.String characterSet)
Sets the character set to use.
|
void |
setConnectionTimeout(int connectionTimeout)
Sets the connection timeout to use.
|
void |
setCookie(ScrapingCookie cookie)
Allows a cookie to be set in the current session state.
|
void |
setCookie(java.lang.String domain,
java.lang.String name,
java.lang.String value)
Allows a cookie to be set in the current session state.
|
void |
setCookie(java.lang.String domain,
java.lang.String name,
java.lang.String value,
java.lang.String path,
java.util.Date expires,
boolean secure)
Sets a cookie
|
void |
setCookiePolicy(int cookiePolicy)
Sets the cookie policy option.
|
void |
setCreatedByVersion(java.lang.String createdByVersion)
Sets the version of ss that originally created the scraping session.
|
void |
setCurrentScriptContext(ScriptContext currentScriptContext)
Sets the current
ScriptContext . |
void |
setCustomScriptVariable(java.lang.String name,
java.lang.Object value)
Sets a custom script variable.
|
void |
setDateExported(java.lang.String date)
Sets the date the scraping session was exported.
|
void |
setDateUpdated(long dateUpdated)
Sets the last time the scraping session was updated.
|
void |
setDebugMode(boolean debugMode)
Sets the debug state for the scrape.
|
void |
setDefaultRetryPolicy(RetryPolicy retryPolicy)
Sets a retry policy that will affect all files in the scrape.
|
void |
setDefaultUserAgent(java.lang.String defaultUserAgent)
Sets the default user agent.
|
void |
setDoLazyScrape(boolean doLazyScrape)
Determines whether or not the session should do a "lazy" scrape.
|
void |
setErrorMessage(java.lang.String errorMessage)
Sets the error message that should be persisted to the database once the scraping session ends.
|
void |
setExternalNTProxyDomain(java.lang.String externalNTProxyDomain)
Sets the external NT proxy port.
|
void |
setExternalNTProxyHost(java.lang.String externalNTProxyHost)
Sets the external NT proxy host.
|
void |
setExternalNTProxyPassword(java.lang.String externalNTProxyPassword)
Sets the external NT proxy password.
|
void |
setExternalNTProxyUsername(java.lang.String externalNTProxyUsername)
Sets the external NT proxy username.
|
void |
setExternalProxyHost(java.lang.String externalProxyHost)
Sets the external proxy host.
|
void |
setExternalProxyPassword(java.lang.String externalProxyPassword)
Sets the external proxy password.
|
void |
setExternalProxyPort(java.lang.String externalProxyPort)
Sets the external proxy port.
|
void |
setExternalProxyUsername(java.lang.String externalProxyUsername)
Sets the external proxy username.
|
void |
setFatalErrorOccurred(boolean fatalErrorOccurred)
Sets the flag used to indicate whether or not a fatal error occurred while running this scraping session.
|
void |
setFinishLock(java.lang.Object finishLock)
Sets the object that should be used to notify external classes when the scraping session finishes.
|
void |
setFolder(Folder folder)
Sets the folder containing this object.
|
void |
setHasBeenQueued(boolean hasBeenQueued)
Sets whether or not the scraping session has been queued to be run.
|
void |
setHttpClient(ScrapingHttpClient httpClient)
Sets the HTTP client to be used in scraping.
|
void |
setHttpClient(ScrapingHttpClient httpClient,
boolean clientWillBeClosedExternally)
Sets the HTTP client to be used in scraping.
|
void |
setHTTPClientType(int httpClientType)
Sets the HTTP client to be used throughout the session.
|
void |
setID(int scrapingSessionID)
Sets the ID of the scraping session.
|
void |
setIdentifier(java.lang.String identifier) |
void |
setInternalVariable(java.lang.String key,
java.lang.Object value)
Sets the value of an internal variable.
|
void |
setIsCompleted(boolean isCompleted)
Sets whether or not the scraper has finished scraping.
|
void |
setIsRunning(boolean isRunning)
Sets whether or not the session is currently running.
|
void |
setKeyStoreFilePath(java.lang.String keyStoreFilePath)
Sets the path to the keystore file.
|
void |
setKeyStorePassword(java.lang.String keyStorePassword)
Sets the password to be used for the key store file.
|
void |
setLastDeterminedCharacterSet(java.lang.String characterSet)
Sets the last character set determined from a server.
|
void |
setLogFileNameAsSessionVariable()
Causes the scraping session to set a session variable corresponding to the name of its log file.
|
void |
setLoggingLevel(int loggingLevel)
Sets the logging level to use when running the scraping session.
|
void |
setLoggingLevel(java.lang.String loggingLevel)
Sets the logging level based on a string.
|
void |
setLoggingLevelDebug()
Convenience method to set logging level to debug
|
void |
setLoggingLevelError()
Convenience method to set logging level to error
|
void |
setLoggingLevelInfo()
Convenience method to set logging level to info
|
void |
setLoggingLevelWarn()
Convenience method to set logging level to warn
|
void |
setMaxConcurrentFileDownloads(int maxConcurrentFileDownloads)
Sets the maximum number of concurrent file downloads to a allow.
|
void |
setMaxHTTPRequests(int maxHTTPRequests)
Sets the maximum number of attempts that should be made for each HTTP request.
|
void |
setMaxScriptsOnStack(int maxScriptsOnStack)
Sets the maximum number of script context objects that can be pushed on to the stack.
|
void |
setName(java.lang.String name)
Sets the name.
|
void |
setNeedsToBeSaved(boolean needsToBeSaved)
Sets whether or not the object needs to be saved.
|
void |
setNode(javax.swing.tree.DefaultMutableTreeNode node)
Sets the node representing this session.
|
void |
setNotes(java.lang.String notes)
Sets the notes.
|
void |
setNotifyNotifiableOfCompletion(boolean notifyNotifiableOfCompletion)
Determines whether or not the notifiable object should be notified once the scraping session completes.
|
void |
setNumberOfRequiredProxies(int numberOfRequiredProxies)
The number of proxies required to anonymize this scraping session
|
void |
setNumDuplicateRecordsScraped(int numRecordsScraped)
Sets the number of duplicate records scraped.
|
void |
setNumErrorRecordsScraped(int numRecordsScraped)
Sets the number of error records scraped.
|
void |
setNumNewRecordsScraped(int numRecordsScraped)
Sets the number of new records scraped.
|
void |
setNumRecordsScraped(int numRecordsScraped)
Sets the number of records scraped for the primary data set corresponding to this scraping session.
|
void |
setNumRecordsScraped(int numRecordsScraped,
boolean addToValue)
Sets the number of records scraped for the primary data set corresponding to this scraping session.
|
void |
setNumRecordsScraped(java.lang.Object numRecordsScraped)
Sets the number of records scraped for the primary data set corresponding to this scraping session.
|
void |
setNumRecordsScraped(java.lang.Object numRecordsScraped,
boolean addToValue)
Sets the number of records scraped for the primary data set corresponding to this scraping session.
|
void |
setNumUpdatedRecordsScraped(int numRecordsScraped)
Sets the number of duplicate records scraped.
|
void |
setOriginatorEdition(int originatorEdition)
Sets the edition of screen-scraper in which the scraping session was created.
|
protected void |
setParentScrapingSession(ScrapingSession parentScrapingSession)
If this scraping session was invoked as a RunnableScrapingSession this sets its parent.
|
void |
setProxyServerPool(ProxyServerPool proxyServerPool)
Gets the current
ProxyServerPool object, used to cycle through a pool of proxies. |
void |
setRandomizeUserAgent(boolean randomizeUserAgent)
Sets whether or not the user agent should be randomized.
|
void |
setRelativeRecordsScrapedThreshold(int relativeRecordsScrapedThreshold)
Sets the records scraped threshold to use for the scraping session.
|
void |
setRelativeTimeThreshold(int relativeTimeThreshold)
Sets the time threshold to use for the scraping session.
|
void |
setResetOnCompletion(boolean resetOnCompletion)
Determines whether or not the session should be reset once it completes.
|
void |
setRetainNonTidiedHTML(boolean retainNonTidiedHTML)
Sets whether or not the tidied HTML should be retained for all of the scrapeable files in this session.
|
void |
setScrapeableFiles(java.util.HashSet scrapeableFiles)
Sets all scrapeable files.
|
(package private) void |
setScrapeCacher(FileCacher cacher)
Sets the cacher that will be used for this session.
|
void |
setScrapingSessionNotifiable(ScrapingSessionNotifiable scrapingSessionNotifiable)
Sets the object the scraping session will use to notify its progress of.
|
void |
setScrapingSessionState(ScrapingSessionState scrapingSessionState)
Explicitly sets the state variable for the session.
|
void |
setScriptInstances(ScriptInstances scriptInstances)
Sets the script instances to be associated with this scrapeable file.
|
void |
setSessionVariablesFromMap(java.util.Map<java.lang.Object,java.lang.Object> map,
boolean ignoreLowerCaseKeys)
Sets the value of all session variables that that match the keys in the Map to the values in the Map.
|
void |
setShouldBreakpoint(boolean shouldBreakpoint)
Sets whether or not the session should breakpoint at the next opportunity
|
void |
setStartTime(long startTime)
Sets the starting time to given value which is in milliseconds.
|
void |
setStatusMessage(java.lang.String statusMessage)
Sets the status message that should be persisted to the database once the scraping session ends.
|
void |
setStopScraping(boolean stopScraping)
Sets whether or not the session should stop scraping.
|
void |
setStopScraping(boolean forceSet,
StopScrapingCallType stopScrapingCallType,
java.lang.String explanation)
Sets whether or not the session should stop scraping.
|
void |
setStopScrapingOnExtractorPatternTimeout(boolean stopScrapingOnExtractorPatternTimeout)
Sets whether or not scraping should stop in the event of an extractor pattern timeout
|
void |
setStopScrapingOnMaxRequestAttemptsReached(boolean stopScrapingOnMaxRequestAttemptsReached)
Sets whether or not scraping should stop in the event of max request attempts reached.
|
void |
setStopScrapingOnScriptError(boolean stopScrapingOnScriptError)
Sets whether or not scraping should stop in the event of a script error.
|
void |
setTerminateProxiesOnCompletion(boolean terminateProxiesOnCompletion)
Indicates whether or not proxies should be terminated on completion
|
void |
setTimeout(int timeout)
Sets the timeout, in minutes, for the scraping session.
|
void |
setTimeZone(java.lang.String timeZone)
Sets the time zone that will be used when using a method that returns a time formatted as a string, but doesn't take a time.
|
void |
setTimeZone(java.util.TimeZone timeZone)
Sets the time zone that will be used when using a method that returns a time formatted as a string, but doesn't take a time.
|
void |
setURLToUseAsReferer(java.net.URL urlToUseAsReferer)
Sets the URL that should be used for the referer HTTP header for scrapeable files subsequent to this one.
|
void |
setUseOnlySSLv3(boolean useOnlySSLv3)
Sets whether or not only SSLv3 should be used when connecting to remote servers.
|
void |
setUseProxyFromPool(boolean useProxyFromPool)
Sets whether or not a proxy should be used for the current request(s).
|
void |
setUserAgent(java.lang.String userAgent)
Sets the user-agent to be used in HTTP requests.
|
void |
setUseServerCharacterSet(boolean useServerCharacterSet)
Sets whether the character set indicated by the server should be used.
|
void |
setUseStrictMode(boolean useStrictMode)
Determines whether or not strict mode should be used.
|
void |
setv(java.lang.String variable,
java.lang.Object value)
Alias to setVariable.
|
void |
setVariable(java.lang.String variable,
java.lang.Object value)
Sets the value of a session variable.
|
void |
setVariables(java.util.Map<java.lang.Object,java.lang.Object> variables)
Sets the value of all session variables that that match the keys in the Map to the values in the Map.
|
void |
setVariables(java.util.Map<java.lang.Object,java.lang.Object> variables,
boolean ignoreLowerCaseKeys)
Sets the value of all session variables that that match the keys in the Map to the values in the Map.
|
protected void |
setWebClient(com.gargoylesoftware.htmlunit.WebClient webClient)
Sets the web client used in navigating.
|
boolean |
shouldStopScraping()
Indicates whether or not the session should stop scraping.
|
void |
startScraping()
Starts the session scraping.
|
void |
stopChildScrapingSessions()
Stops any running scraping session objects.
|
void |
stopScraping()
Indicates that the scraping session should stop scraping as soon as possible.
|
void |
stopScraping(StopScrapingCallType callType,
java.lang.String explanation)
Indicates that the scraping session should stop scraping as soon as possible.
|
void |
terminateAllAnonymousProxies()
Stops all of the currently running proxy instances
|
java.lang.String |
toString()
Overridden so that the name appears as the title in the main tree.
|
(package private) void |
unlockBreakpointBlockedThreads()
This should be called at the end of the scrape.
|
static ScrapingSession |
unmarshalScrapingSession(java.io.Reader reader)
Unmarshals a scraping session from the given reader.
|
void |
validateSequences()
Checks each
ScrapeableFile for duplicate sequences. |
void |
waitForBreakpoint()
Waits for a breakpoint in the workbench.
|
void |
waitForBreakpoint(boolean pauseBackgroundThreads)
Deprecated.
pauseBackgroundThreads was causing some locking issues, so it is now ignored
|
void |
waitForFileDownloadsToComplete()
Waits for any file downloads to complete before returning.
|
void |
warn(java.lang.Object message)
Sends an app event off to the view to be logged as WARN.
|
void |
warn(java.lang.Object message,
java.lang.Throwable throwable)
Sends an app event off to the view to be logged as WARN.
|
(package private) boolean |
wasStoppedDuringRun() |
void |
webClose(java.lang.Object message)
Logs closing values to indicate the scrape is complete and what values were when everything finished.
|
void |
webClose(java.lang.Object message,
java.lang.Object loggable)
Logs an error message to the web interface error message area.
|
void |
webDebug(java.lang.Object message)
Logs a debug message to the web interface error message area.
|
void |
webDebug(java.lang.Object message,
boolean saveMessage)
Logs a debug message to the web interface error message area.
|
void |
webDebug(java.lang.Object message,
boolean saveMessage,
java.lang.Object loggable)
Logs a debug message to the web interface error message area.
|
void |
webDebug(java.lang.Object message,
java.lang.Object loggable)
Logs a debug message to the web interface error message area.
|
void |
webError(java.lang.Object message)
Logs an error message to the web interface error message area.
|
void |
webError(java.lang.Object message,
boolean saveMessage)
Logs an error message to the web interface error message area.
|
void |
webError(java.lang.Object message,
boolean saveMessage,
java.lang.Object loggable)
Logs an error message to the web interface error message area.
|
void |
webError(java.lang.Object message,
java.lang.Object loggable)
Logs an error message to the web interface error message area.
|
void |
webInfo(java.lang.Object message)
Logs an info message to the web interface error message area.
|
void |
webInfo(java.lang.Object message,
boolean saveMessage)
Logs an info message to the web interface error message area.
|
void |
webInfo(java.lang.Object message,
boolean saveMessage,
java.lang.Object loggable)
Logs an info message to the web interface error message area.
|
void |
webInfo(java.lang.Object message,
java.lang.Object loggable)
Logs an info message to the web interface error message area.
|
void |
webWarn(java.lang.Object message)
Logs a warn message to the web interface error message area.
|
void |
webWarn(java.lang.Object message,
boolean saveMessage)
Logs a warn message to the web interface error message area.
|
void |
webWarn(java.lang.Object message,
boolean saveMessage,
java.lang.Object loggable)
Logs a warn message to the web interface error message area.
|
void |
webWarn(java.lang.Object message,
java.lang.Object loggable)
Logs a warn message to the web interface error message area.
|
public static org.apache.log4j.Logger log
public static final java.lang.String STRING_ACCORDING_TO_COOKIE_SPEC
public static final java.lang.String STRING_ACCEPT
public static final java.lang.String STRING_BEST_MATCH
public static final java.lang.String STRING_COMPATIBILITY
public static final java.lang.String STRING_NETSCAPE_DRAFT
public static final java.lang.String STRING_REJECT
public static final java.lang.String STRING_STRICT_MODE
public static final int INT_ACCEPT
public static final int INT_REJECT
public static final int INT_BEST_MATCH
public static final int INT_STRICT_MODE
public static final java.lang.String STRING_APACHE_HTTPCLIENT
public static final java.lang.String STRING_NING_ASYNC_HTTPCLIENT
public static final java.lang.String STRING_ASYNC_HTTPCLIENT_2
public static final java.lang.String STRING_CURL_HTTPCLIENT
public static final int INT_APACHE_HTTPCLIENT
public static final int INT_NING_ASYNC_HTTPCLIENT
public static final int INT_ASYNC_HTTPCLIENT_2
public static final int INT_CURL_HTTPCLIENT
public static final java.lang.String STRING_HTTP_CLIENT_HTTPCLIENT
public static final java.lang.String STRING_HTTP_CLIENT_MOZILLA
public static final java.lang.String STRING_HTTP_CLIENT_IE
public static final int INT_HTTP_CLIENT_HTTPCLIENT
public static final int INT_HTTP_CLIENT_MOZILLA
public static final int INT_HTTP_CLIENT_IE
public static final java.awt.datatransfer.DataFlavor INFO_FLAVOR
@NotNull static java.awt.datatransfer.DataFlavor[] flavors
public ScrapingSession()
public ScrapingSession(@NotNull ScrapingSession scrapingSession)
scrapingSession
- The ScrapingSession
to be copied.public ScrapingSession(java.lang.String name)
name
- The name of the session.public ScrapingSession(int scrapingSessionID, java.lang.String name, int cookieHandling, int cookiePolicy, int httpClientType, int maxHTTPRequests, java.lang.String externalProxyUsername, java.lang.String externalProxyPassword, java.lang.String externalProxyHost, java.lang.String externalProxyPort, java.lang.String externalNTProxyUsername, java.lang.String externalNTProxyPassword, java.lang.String externalNTProxyHost, java.lang.String externalNTProxyPort, boolean useStrictMode, int folderID, java.lang.String notes, boolean anonymize, boolean terminateProxiesOnCompletion, int numberOfRequiredProxies, int originatorEdition, int loggingLevel, java.lang.String characterSet, java.lang.String keyStoreFilePath, java.lang.String keyStorePassword, boolean useOnlySSLv3, java.lang.String createdByVersion, long dateUpdated)
anonymize
- Indicates whether the scraping session should be anonymizedterminateProxiesOnCompletion
- Indicates whether the proxies should be terminatednumberOfRequiredProxies
- The number of required proxies for the scrapeoriginatorEdition
- The edition from which the scraping session originated.loggingLevel
- The logging level to use when the scraping session is running.scrapingSessionID
- The ID of the scraping session.name
- The name of the session.cookieHandling
- Indicates how cookies should be handled.cookiePolicy
- Indicates the cookie policy.httpClientType
- Indicates the http-client.maxHTTPRequests
- The maximum HTTP requests.externalProxyUsername
- The external proxy username.externalProxyPassword
- The external proxy password.externalProxyHost
- The external proxy host.externalProxyPort
- The external proxy port.externalNTProxyUsername
- The external NT proxy username.externalNTProxyPassword
- The external NT proxy password.externalNTProxyHost
- The external NT proxy host.externalNTProxyPort
- The external NT proxy port.useStrictMode
- Whether or not strict mode should be used.folderID
- The ID of the folder containing the object.notes
- The notes.characterSet
- The character set to use.dateUpdated
- The time when the scraping session was last updated.keyStoreFilePath
- The path to the keystore file.keyStorePassword
- The keystore password.useOnlySSLv3
- Whether or not only SSLv3 should be used.createdByVersion
- The version of ss that created this scraping session.@NotNull @InternalOnly public static ScrapingSession unmarshalScrapingSession(java.io.Reader reader) throws org.exolab.castor.mapping.MappingException, java.io.IOException, org.exolab.castor.xml.ValidationException, org.exolab.castor.xml.MarshalException
reader
- A Reader
stream containing the scraping session to be unmarshalled.org.exolab.castor.mapping.MappingException
java.io.IOException
org.exolab.castor.xml.ValidationException
org.exolab.castor.xml.MarshalException
@InternalOnly public void prepareAnonymization()
public void terminateAllAnonymousProxies()
RunningScrapingSession
terminateAllAnonymousProxies
in interface RunningScrapingSession
public boolean doLazyScrape()
public void setDoLazyScrape(boolean doLazyScrape)
doLazyScrape
- A boolean.@InternalOnly public int getHTTPClientType()
@InternalOnly public void setHTTPClientType(int httpClientType)
httpClientType
- The client as defined by one of the "HTTP" constants.@InternalOnly public static int mapHTTPClientTypeStringToInt(@NotNull java.lang.String httpClientType)
httpClientType
- The string representing the HTTP client.@NotNull @InternalOnly public static java.lang.String mapHTTPClientIntToString(int httpClient)
httpClient
- The int representing the HTTP client.@RequiredVersion(value=1) public int getMaxConcurrentFileDownloads()
@RequiredVersion(value=1) public void setMaxConcurrentFileDownloads(int maxConcurrentFileDownloads)
maxConcurrentFileDownloads
- The maximum number of downloads.@RequiredVersion(value=2) public boolean downloadFile(java.lang.String urlString, java.lang.String fileName, int maxNumAttempts, boolean doLazy)
fileName
- The full path that the downloaded resource should be written to. Example "C:\\downloaded.gif". The "\" character should be included in the path
twice for each instance of it's use in order for it to be not treated as an escape character.urlString
- The Uniform Resource Locator of the resource.maxNumAttempts
- The number of attempts the download should be attempted in the case of failures.doLazy
- Indicates whether or not the file should be downloaded in its own thread.@RequiredVersion(value=1) public boolean downloadFile(java.lang.String urlString, java.lang.String fileName, int maxNumAttempts)
fileName
- The full path that the downloaded resource should be written to. Example "C:\\downloaded.gif". The "\" character should be included in the path
twice for each instance of it's use in order for it to be not treated as an escape character.urlString
- The Uniform Resource Locator of the resource.maxNumAttempts
- The number of attempts the download should be attempted in the case of failures.@RequiredVersion(value=1) public boolean downloadFile(@Nullable java.lang.String urlString, @Nullable java.lang.String fileName)
downloadFile
in interface RunningScrapingSession
fileName
- The full path that the downloaded resource should be written to. Example "C:\\downloaded.gif". The "\" character should be included in the path
twice for each instance of it's use in order for it to be not treated as an escape character.urlString
- The Uniform Resource Locator of the resource.@RequiredVersion(value=2, behavior=LOG_ERROR_ONLY) public void waitForFileDownloadsToComplete()
@InternalOnly public void setID(int scrapingSessionID)
scrapingSessionID
- The id of the scraping session@InternalOnly public int getID()
getID
in interface ScriptInstancesContainer
@Nullable public java.lang.String getName()
getName
in interface ScriptInstancesContainer
public void setName(java.lang.String name)
name
- The name.@Nullable @InternalOnly public java.lang.String getNameUpperCase()
@Nullable @InternalOnly public Folder getFolder()
getFolder
in interface Folderable
@InternalOnly public void setFolder(@Nullable Folder folder)
setFolder
in interface Folderable
folder
- The folder.@InternalOnly public void removeFromFolder()
removeFromFolder
in interface Folderable
@InternalOnly public void moveToNewFolder(Folder folder)
moveToNewFolder
in interface Folderable
folder
- The Folder
to which the object should be moved.@InternalOnly public int getCookiePolicy()
@InternalOnly public java.lang.String getCookiePolicyAsHttpClientString()
String
that is friendly to HttpClient 4.@InternalOnly public void setCookiePolicy(int cookiePolicy)
cookiePolicy
- The option, as an int.@InternalOnly public static int mapCookiePolicyHumanReadableStringToInt(@NotNull java.lang.String cookiePolicy)
cookiePolicy
- The string representing the cookie policy.@NotNull @InternalOnly public static java.lang.String mapCookiePolicyIntToHumanReadableString(int cookiePolicy)
cookiePolicy
- The int representing the cookie policy.@InternalOnly public void setNode(javax.swing.tree.DefaultMutableTreeNode node)
node
- A DefaultMutableTreeNode
.@Nullable @InternalOnly public javax.swing.tree.DefaultMutableTreeNode getNode()
DefaultMutableTreeNode
.@InternalOnly public java.lang.String getNextDefaultScrapeableFileName()
@InternalOnly public int getNumScrapeableFiles()
@InternalOnly public int getNumScrapeableFilesWithSequence()
@InternalOnly public void resetInitialValues()
@InternalOnly public void resetContainsNullableTokens()
@NotNull @Deprecated @InternalOnly public org.apache.commons.httpclient.Cookie[] getCookies()
getCookieList()
method instead.Cookie
objects corresponding to the current session.Cookie
objects.@NotNull public java.util.List<ScrapingCookie> getCookieList()
@Nullable public ScrapingHttpClient getHttpClient()
CloseableHttpClient
.public void setHttpClient(@Nullable ScrapingHttpClient httpClient)
httpClient
- A CloseableHttpClient
.public void setHttpClient(@Nullable ScrapingHttpClient httpClient, boolean clientWillBeClosedExternally)
httpClient
- A CloseableHttpClient
.clientWillBeClosedExternally
- True if the client being passed in will be closed someplace else, so should not be automatically closed@InternalOnly @NotNull public com.gargoylesoftware.htmlunit.WebClient getWebClient()
WebClient
object.protected void setWebClient(@Nullable com.gargoylesoftware.htmlunit.WebClient webClient)
webClient
- The WebClient
object.@InternalOnly public void addScrapeableFile(@Nullable ScrapeableFile scrapeableFile)
scrapeableFile
- A ScrapeableFile
.@InternalOnly public void removeScrapeableFile(@Nullable java.lang.String name)
name
- The name of the scrapeable file to be removed.@InternalOnly public void removeScrapeableFile(int sequence)
sequence
- The sequence of the scrapeable file to be removed.@InternalOnly public void replaceScrapeableFile(@Nullable ScrapeableFile scrapeableFile)
scrapeableFile
- The ScrapeableFile
to replace.@InternalOnly public void bumpScrapeableFileSequences(int minimumSequence, int maximumSequence, boolean bumpUp)
ScrapeableFile
whose sequence falls in between the given sequences.minimumSequence
- The minimum sequence to compare with.maximumSequence
- The maximum sequence to compare with.bumpUp
- Indicates that sequences should be bumped up.@InternalOnly public void validateSequences()
ScrapeableFile
for duplicate sequences. If a duplicate sequence is found, the scrapeable file is given no sequence. This method also
ensures there aren't any holes in the sequences.@InternalOnly public java.util.Iterator getScrapeableFilesIterator()
Iterator
used to cycle through the ScrapeableFile
objects held by this object.Iterator
.@Nullable @InternalOnly public java.util.HashSet getScrapeableFiles()
ScrapeableFile
s associated with this scraping session. Note that this method is only public because of Castor.ScrapeableFile
s as a HashSet
.@InternalOnly public void setScrapeableFiles(java.util.HashSet scrapeableFiles)
scrapeableFiles
- A HashSet
of ScrapeableFile
s.@InternalOnly public ScrapeableFile getScrapeableFile(int sequence)
sequence
- The sequence of the scrapeable file to be retrieved.@InternalOnly public ScrapeableFile getScrapeableFileByID(int scrapeableFileID)
scrapeableFileID
- The ID of the scrapeable file to be retrieved.@InternalOnly public ScrapeableFile getNonSequencedScrapeableFileBySequence(int sequence)
sequence
- The sequence of the scrapeable file to be retrieved.@Nullable @InternalOnly public ScrapeableFile getScrapeableFile(@Nullable java.lang.String name)
name
- The name of the scrapeable file to be retrieved.@InternalOnly public void ensureScrapeableFilesHaveScrapingSessionReference()
@Nullable @InternalOnly public ScrapeableFile getScrapeableFileCaseInsensitive(java.lang.String name)
name
- The name of the scrapeable file to be retrieved.@Nullable public java.lang.String toString()
toString
in class java.lang.Object
@InternalOnly public ScriptInstances getScriptInstances()
ScriptInstances
object containing all script instances associated with this scrapeable file.getScriptInstances
in interface ScriptInstancesContainer
@InternalOnly public void setScriptInstances(ScriptInstances scriptInstances)
setScriptInstances
in interface ScriptInstancesContainer
scriptInstances
- The script instances.@InternalOnly public void ensureFullyLoaded()
Note: this method is synchronized because of a case we encountered where the scraping session was being copied by multiple scraping sessions that all invoked it. Some of the scraping sessions would start copying it before it finished loading.
@InternalOnly public void save()
@RequiredVersion(value=1) public void setCookie(@NotNull java.lang.String domain, @NotNull java.lang.String name, @NotNull java.lang.String value)
setCookie
in interface RunningScrapingSession
domain
- The domain of the cookie.name
- The name of the cookie.value
- The value the cookie should hold.@RequiredVersion(value=1) public void setCookie(@NotNull java.lang.String domain, @NotNull java.lang.String name, @NotNull java.lang.String value, java.lang.String path, @Nullable java.util.Date expires, boolean secure)
name
- the cookie namevalue
- the cookie valuedomain
- the domain this cookie can be sent topath
- the path prefix for which this cookie can be sentexpires
- the Date
at which this cookie expires, or null if the cookie expires at the end of the sessionsecure
- if true this cookie can only be sent over secure connectionsjava.lang.IllegalArgumentException
- If cookie name is null or blank, cookie name contains a blank, or cookie name starts with character $@RequiredVersion(value=1) public void setCookie(@NotNull ScrapingCookie cookie)
cookie
- The cookie to be set@RequiredVersion(value=2) public void clearCookies()
clearCookies
in interface RunningScrapingSession
@InternalOnly public void delete()
delete
in interface Deleteable
@InternalOnly public void startScraping()
@RequiredVersion(value=1) public void executeScript(java.lang.String scriptName) throws java.lang.Exception
executeScript
in interface RunningScrapingSession
scriptName
- The name of the script to execute.java.lang.Exception
- if the script couldn't be foundprotected void processPreScrapingScripts()
protected void processPostScrapingScripts()
@Nullable public java.lang.Object getVariable(@NotNull java.lang.String variable)
getVariable
in interface RunningScrapingSession
variable
- The name of the variable.@InternalOnly @Nullable public <T> T getInternalVariable(@NotNull java.lang.String key)
key
- The name of the variable.@Nullable public java.lang.Object getv(@NotNull java.lang.String variable)
variable
- The name of the variable.@NotNull protected java.lang.String getAllSessionVariablesAsURLEncodedString()
public void setVariable(@NotNull java.lang.String variable, @Nullable java.lang.Object value)
setVariable
in interface RunningScrapingSession
variable
- The variable.value
- The value of the variable.@InternalOnly public void setInternalVariable(@NotNull java.lang.String key, @Nullable java.lang.Object value)
key
- The variable.value
- The value of the variable.public void setv(@NotNull java.lang.String variable, @Nullable java.lang.Object value)
variable
- The variable.value
- The value of the variable.@Deprecated public void addSessionVariableMonitor(java.lang.String monitoredVariable)
addMonitoredVariable(String)
instead.monitoredVariable
- The name of the monitored variablepublic void clearAllSessionVariables()
public void convertHTMLEntitiesInVariable(@NotNull java.lang.String variable)
variable
- The name of the variable.@RequiredVersion(value=2) public void loadVariables(@NotNull java.lang.String fileToReadFrom)
fileToReadFrom
- The file to read the variables from.@RequiredVersion(value=2) public void saveVariables(@NotNull java.lang.String fileToSaveTo)
saveVariables
in interface RunningScrapingSession
fileToSaveTo
- The file to save the variables to.@RequiredVersion(value=1) public void addToVariable(@NotNull java.lang.String variable, int value)
addToVariable
in interface RunningScrapingSession
variable
- The name of the session variable.value
- The value to add.@NotNull @InternalOnly public ScrapingSessionState getScrapingSessionState()
@InternalOnly public void setScrapingSessionState(@Nullable ScrapingSessionState scrapingSessionState)
scrapingSessionState
- The ScrapingSessionState
@InternalOnly public void resetScrapingSessionState()
@InternalOnly public void setScrapingSessionNotifiable(@Nullable ScrapingSessionNotifiable scrapingSessionNotifiable)
scrapingSessionNotifiable
- The object.@Nullable @InternalOnly public ScrapingSessionNotifiable getScrapingSessionNotifiable()
getScrapingSessionNotifiable
in interface RunningScrapingSession
public void log(int message)
message
- The message to be logged.public void log(java.lang.Object message)
log
in interface RunningScrapingSession
message
- The message to be logged.public void logDebug(java.lang.Object message) throws java.lang.Exception
message
- The message to be logged.java.lang.Exception
@RequiredVersion(value=1) public void logDebug(java.lang.String message)
message
- The message to be logged.public void logInfo(java.lang.Object message)
message
- The message to be logged.public void logInfo(java.lang.String message)
message
- The message to be logged.public void logWarn(java.lang.String message)
message
- The message to be logged.public void logWarn(java.lang.Object message)
message
- The message to be logged.public void logError(java.lang.String message)
message
- The message to be logged.public void logError(java.lang.Object message)
message
- The message to be logged.public void debug(java.lang.Object message)
message
- The message to be loggedpublic void debug(java.lang.Object message, java.lang.Throwable throwable)
message
- The message to be loggedthrowable
- A throwable to log with the messagepublic void info(java.lang.Object message)
message
- The message to be loggedpublic void info(java.lang.Object message, java.lang.Throwable throwable)
message
- The message to be loggedthrowable
- A throwable to log with the messagepublic void warn(java.lang.Object message)
message
- The message to be loggedpublic void warn(java.lang.Object message, java.lang.Throwable throwable)
message
- The message to be loggedthrowable
- A throwable to log with the messagepublic void error(java.lang.Object message)
message
- The message to be loggedpublic void error(java.lang.Object message, java.lang.Throwable throwable)
message
- The message to be loggedthrowable
- A throwable to log with the message@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logError(java.lang.Object message, @NotNull java.lang.Throwable t)
message
- The message to be logged.public void logVariables() throws java.lang.Exception
java.lang.Exception
@RequiredVersion(value=1) public void logVariables(@Nullable ScriptContext scriptContext)
scriptContext
- So that local variables can be logged.@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void scrapeString(java.lang.String scrapeableFileName, java.lang.String content) throws java.lang.Exception
scrapeableFileName
- The name of the file to scrapecontent
- The content to use as the last responsejava.lang.Exception
- If an error occurs trying to load the scrapeable filepublic void scrapeString(@Nullable ScrapeableFile scrapeableFile, java.lang.String content)
scrapeableFile
- The scrapeable file to scrapecontent
- The content to use for the last responsepublic void scrapeFile(@Nullable ScrapeableFile scrapeableFile)
scrapeableFile
- The file to be scraped.@InternalOnly public int getNumActiveScrapeableFiles()
@InternalOnly public ScrapeableFile getLatestActiveScrapeableFile()
ScrapeableFile
.@Nullable public java.lang.String getCreatedByVersion()
public void setCreatedByVersion(java.lang.String createdByVersion)
createdByVersion
- The version number.public boolean isCreatedByVersionAtLeast(@NotNull java.lang.String compareToVersion)
compareToVersion
- The version to compare to.@Nullable public java.net.URL getURLToUseAsReferer()
@InternalOnly public void setURLToUseAsReferer(java.net.URL urlToUseAsReferer)
urlToUseAsReferer
- The URL to use as a referer.public void scrapeFile(java.lang.String scrapeableFileName) throws java.lang.Exception
scrapeFile
in interface RunningScrapingSession
scrapeableFileName
- The name of the file to be scraped.java.lang.Exception
public void pause(long time) throws java.lang.Exception
pause
in interface RunningScrapingSession
time
- The amount of time to pause, in milliseconds.java.lang.Exception
@InternalOnly @Nullable public ScriptContext getCurrentScriptContext()
ScriptContext
. Used by the BreakpointFrame
.ScriptContext
object.@InternalOnly public void setCurrentScriptContext(ScriptContext currentScriptContext)
ScriptContext
. Used by the BreakpointFrame
.currentScriptContext
- A ScriptContext
object.public void breakpoint()
breakpoint
in interface RunningScrapingSession
@InternalOnly public void breakpoint(@Nullable ScriptContext scriptContext)
scriptContext
- The ScriptContext
whose information should be displayed.public void sendMail(java.lang.String subject, java.lang.String body, java.lang.String recipients, java.lang.String attachments, java.lang.String headers) throws java.lang.Exception
sendMail
in interface RunningScrapingSession
subject
- The subject line of the e-mail.body
- The text of the body of the e-mail.recipients
- A comma separated list of recipients.attachments
- A comma separated list of local files to be attached.headers
- A tab separated list of additional headers one would like to add to the e-mail.java.lang.Exception
public void stopScraping()
stopScraping
in interface RunningScrapingSession
public void stopScraping(@NotNull StopScrapingCallType callType, @Nullable java.lang.String explanation)
stopScraping
in interface RunningScrapingSession
callType
- The type of stop being issuedexplanation
- An explanation string for the stop@RequiredVersion(value=1) public <T extends AbstractEventData> void addEventCallback(@NotNull EventFireTime<T> eventTime, @NotNull EventHandler<T> callback)
eventTime
- The time to execute the callbackcallback
- The callback to execute@RequiredVersion(value=1) public <T extends AbstractEventData> void addEventCallbackWithPriority(@NotNull EventFireTime<T> eventTime, @NotNull EventHandler<T> callback, int priority)
eventTime
- The time to execute the callbackcallback
- The callback to executepriority
- The priority for this callback. Lower numbers are higher priority.@RequiredVersion(value=1) public <T extends AbstractEventData> void removeEventCallback(@NotNull EventFireTime<T> eventTime, @NotNull EventHandler<T> callback)
eventTime
- The time to execute the callbackcallback
- The callback to remove@RequiredVersion(value=1) public <T extends AbstractEventData> void clearCallbacksForTime(@NotNull EventFireTime<T> eventTime)
eventTime
- The time to clear@InternalOnly public boolean getShouldBreakpoint()
@InternalOnly public void setShouldBreakpoint(boolean shouldBreakpoint)
shouldBreakpoint
- Whether or not the session should breakpoint at the next opportunity@Nullable public StopScrapingCallType getStopScrapingCallType()
@Nullable public java.lang.String getStopScrapingExplanation()
public boolean shouldStopScraping()
public void breakpointIfNeeded()
public void setStopScraping(boolean stopScraping)
stopScraping
- A boolean.public void setStopScraping(boolean forceSet, @Nullable StopScrapingCallType stopScrapingCallType, @Nullable java.lang.String explanation)
forceSet
- True if this call should update the values even if we are already set to stop, false to only override an unset valuestopScrapingCallType
- The type of call issued to stop scraping (or null to indicate nothing -- in which case forceSet must also be true)explanation
- The reason, as a string, why the scrape was stopped. This can be set even if null was passed for the type (which means there is a reason to
stop,
but no stop has actually been requested)@InternalOnly boolean wasStoppedDuringRun()
@InternalOnly public void setIsRunning(boolean isRunning)
isRunning
- A boolean.@InternalOnly public boolean isRunning()
@InternalOnly public boolean getHasBeenQueued()
@InternalOnly public void setHasBeenQueued(boolean hasBeenQueued)
hasBeenQueued
- A boolean.@InternalOnly public boolean isCompleted()
@InternalOnly public void setIsCompleted(boolean isCompleted)
isCompleted
- A boolean.@InternalOnly public void setFinishLock(java.lang.Object finishLock)
finishLock
- The object to use.@InternalOnly public boolean getNotifyNotifiableOfCompletion()
@InternalOnly public void setNotifyNotifiableOfCompletion(boolean notifyNotifiableOfCompletion)
notifyNotifiableOfCompletion
- A boolean.@InternalOnly public void scrape()
public int getTimeout()
public long getStartTime()
public void setStartTime(long startTime)
startTime
- The starting time in milliseconds.public void setTimeout(int timeout)
timeout
- The timeout, in minutes.@InternalOnly public void markStopTime()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public long getElapsedRunningTime()
getElapsedRunningTime
in interface RunningScrapingSession
@RequiredVersion(value=1) public void logElapsedRunningTime()
logElapsedRunningTime
in interface RunningScrapingSession
@RequiredVersion(value=1) public void logCurrentTime()
logCurrentTime
in interface RunningScrapingSession
@RequiredVersion(value=1) public void logCurrentDateAndTime()
logCurrentDateAndTime
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public boolean getFatalErrorOccurred()
getFatalErrorOccurred
in interface RunningScrapingSession
@RequiredVersion(value=2) public void setFatalErrorOccurred(boolean fatalErrorOccurred)
setFatalErrorOccurred
in interface RunningScrapingSession
fatalErrorOccurred
- A boolean.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.String getErrorMessage()
getErrorMessage
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.String getStatusMessage()
@RequiredVersion(value=2) public void setErrorMessage(@Nullable java.lang.String errorMessage)
setErrorMessage
in interface RunningScrapingSession
errorMessage
- The error message to set.@RequiredVersion(value=2) public void setStatusMessage(java.lang.String statusMessage)
statusMessage
- The status message to set.@RequiredVersion(value=2) public void appendErrorMessage(java.lang.String errorMessage)
errorMessage
- The error message to append.@RequiredVersion(value=2) public void appendStatusMessage(java.lang.String statusMessage)
statusMessage
- The status message to append.@RequiredVersion(value=2) public void setNumRecordsScraped(java.lang.Object numRecordsScraped)
setNumRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of records scraped. passed in will be added to the current value rather than overwriting it.public void resetNumRecordsScraped()
@RequiredVersion(value=2) public void addToNumRecordsScraped(java.lang.Object numRecordsScraped)
addToNumRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of records to add.public void setNumRecordsScraped(java.lang.Object numRecordsScraped, boolean addToValue)
numRecordsScraped
- The number of records scraped.addToValue
- If this value is true, the number of records passed in will be added to the current value rather than overwriting it.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addToNumRecordsScraped(int numRecordsScraped)
addToNumRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of records scraped.public void setNumRecordsScraped(int numRecordsScraped)
setNumRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of records scraped.public void setNumRecordsScraped(int numRecordsScraped, boolean addToValue)
numRecordsScraped
- The number of records scraped.addToValue
- If this value is true, the number of records passed in will be added to the current value rather than overwriting it.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addToNumNewRecordsScraped(int numRecordsScraped)
RunningScrapingSession
addToNumNewRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of new records scraped to add.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addToNumDuplicateRecordsScraped(int numRecordsScraped)
RunningScrapingSession
addToNumDuplicateRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of duplicate records scraped to add.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addToNumUpdatedRecordsScraped(int numRecordsScraped)
RunningScrapingSession
addToNumUpdatedRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of updated records scraped to add.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addToNumErrorRecordsScraped(int numRecordsScraped)
RunningScrapingSession
addToNumErrorRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of error records scraped to add.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setNumNewRecordsScraped(int numRecordsScraped)
RunningScrapingSession
setNumNewRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of new records scraped.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setNumDuplicateRecordsScraped(int numRecordsScraped)
RunningScrapingSession
setNumDuplicateRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of duplicate records scraped.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setNumUpdatedRecordsScraped(int numRecordsScraped)
RunningScrapingSession
setNumUpdatedRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of duplicate records scraped.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setNumErrorRecordsScraped(int numRecordsScraped)
RunningScrapingSession
setNumErrorRecordsScraped
in interface RunningScrapingSession
numRecordsScraped
- The number of error records scraped.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumNewRecordsScraped()
RunningScrapingSession
getNumNewRecordsScraped
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumDuplicateRecordsScraped()
RunningScrapingSession
getNumDuplicateRecordsScraped
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumUpdatedRecordsScraped()
RunningScrapingSession
getNumUpdatedRecordsScraped
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumErrorRecordsScraped()
RunningScrapingSession
getNumErrorRecordsScraped
in interface RunningScrapingSession
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumRecordsScraped()
getNumRecordsScraped
in interface RunningScrapingSession
@InternalOnly public int getRelativeTimeThreshold()
@InternalOnly public void setRelativeTimeThreshold(int relativeTimeThreshold)
relativeTimeThreshold
- The threshold percentage, as an integer.@InternalOnly public int getRelativeRecordsScrapedThreshold()
@InternalOnly public void setRelativeRecordsScrapedThreshold(int relativeRecordsScrapedThreshold)
relativeRecordsScrapedThreshold
- The threshold percentage, as an integer.@InternalOnly public boolean sessionTimedOut()
@InternalOnly public boolean resetOnCompletion()
@InternalOnly public void setResetOnCompletion(boolean resetOnCompletion)
resetOnCompletion
- A boolean.@InternalOnly public void resetReferers()
protected void finalize() throws java.lang.Throwable
finalize
in class java.lang.Object
java.lang.Throwable
@InternalOnly public void doExtendedReset(boolean nullScrapingSessionNotifiables)
nullScrapingSessionNotifiables
- Indicates whether or not the ScrapingSessionNotifiable objects need to be nulled out.public void clearCustomScriptVariables()
@NotNull public java.util.Map<java.lang.String,java.lang.Object> getCustomScriptVariables()
@RequiredVersion(value=1) @Nullable public java.lang.Object getCustomScriptVariable(@NotNull java.lang.String name)
name
- The name of the variable@RequiredVersion(value=1) public void setCustomScriptVariable(@NotNull java.lang.String name, @Nullable java.lang.Object value)
name
- The name of the variablevalue
- The value of the variable, or null to clear it@InternalOnly public void flagAsLoaded()
public int getMaxHTTPRequests()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setMaxHTTPRequests(int maxHTTPRequests)
maxHTTPRequests
- The maximum attempts.@Nullable public java.lang.String getExternalProxyUsername()
public void setExternalProxyUsername(@Nullable java.lang.String externalProxyUsername)
externalProxyUsername
- The username.@Nullable public java.lang.String getExternalProxyPassword()
public void setExternalProxyPassword(@Nullable java.lang.String externalProxyPassword)
externalProxyPassword
- The password.@Nullable public java.lang.String getExternalProxyHost()
public void setExternalProxyHost(@Nullable java.lang.String externalProxyHost)
externalProxyHost
- The host.@Nullable public java.lang.String getExternalProxyPort()
public void setExternalProxyPort(@Nullable java.lang.String externalProxyPort)
externalProxyPort
- The port.@Nullable public java.lang.String getExternalNTProxyUsername()
public void setExternalNTProxyUsername(@Nullable java.lang.String externalNTProxyUsername)
externalNTProxyUsername
- The username.@Nullable public java.lang.String getExternalNTProxyPassword()
public void setExternalNTProxyPassword(@Nullable java.lang.String externalNTProxyPassword)
externalNTProxyPassword
- The password.@Nullable public java.lang.String getExternalNTProxyHost()
public void setExternalNTProxyHost(@Nullable java.lang.String externalNTProxyHost)
externalNTProxyHost
- The host.@Nullable public java.lang.String getExternalNTProxyDomain()
public void setExternalNTProxyDomain(@Nullable java.lang.String externalNTProxyDomain)
externalNTProxyDomain
- The port.@InternalOnly public boolean getUseStrictMode()
@InternalOnly public void setUseStrictMode(boolean useStrictMode)
useStrictMode
- A boolean.public boolean getTerminateProxiesOnCompletion()
public void setTerminateProxiesOnCompletion(boolean terminateProxiesOnCompletion)
terminateProxiesOnCompletion
- A boolean.public boolean getAnonymize()
public void setAnonymize(boolean anonymize)
anonymize
- A boolean.public int getNumberOfRequiredProxies()
public void setNumberOfRequiredProxies(int numberOfRequiredProxies)
numberOfRequiredProxies
- An int@InternalOnly public void clearLastRequestsAndResponses()
public void addXMLFeedItem(java.lang.String title, java.lang.String link, java.lang.String description, @Nullable java.util.Date publishedDate)
title
- The title for the feed.link
- The link/URL.description
- The description.publishedDate
- The date the feed was published.public void setUseProxyFromPool(boolean useProxyFromPool)
useProxyFromPool
- A boolean.public boolean getUseProxyFromPool()
@Nullable public ProxyServerPool getProxyServerPool()
ProxyServerPool
object, used to cycle through a pool of proxies.ProxyServerPool
object.public void setProxyServerPool(@Nullable ProxyServerPool proxyServerPool)
ProxyServerPool
object, used to cycle through a pool of proxies.proxyServerPool
- The pool@Nullable public ProxyServer getCurrentProxyServerFromPool()
ProxyServer
being used from the pool.ProxyServer
.@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void currentProxyServerIsBad()
ProxyServer
from the pool.@Nullable @InternalOnly public java.lang.String getNotes()
@InternalOnly public void setNotes(java.lang.String notes)
notes
- The notes.public void setLogFileNameAsSessionVariable()
@Nullable @RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.String getLogFileName()
public boolean getRetainNonTidiedHTML()
@RequiredVersion(value=2) public void setRetainNonTidiedHTML(boolean retainNonTidiedHTML)
retainNonTidiedHTML
- A boolean.@InternalOnly public void export(@NotNull java.io.OutputStream outputStream) throws java.lang.Exception
outputStream
- The OutputStream
the XML will be written to.java.lang.Exception
@InternalOnly public void addAllExecuteScriptScriptsToExport()
@Nullable @InternalOnly public java.util.HashSet getExecuteScriptScriptsToExport()
Script
objects.@InternalOnly public void addExecuteScriptScriptToExport(Script script)
script
- The Script
to be exported.@InternalOnly public int compareTo(java.lang.Object object)
Comparable
interface.compareTo
in interface java.lang.Comparable
object
- The object to compare to.@InternalOnly public int hashCode()
hashCode
in class java.lang.Object
@InternalOnly public boolean equals(@Nullable java.lang.Object object)
equals
in class java.lang.Object
object
- The object we're comparing.@InternalOnly public static boolean readerPointsToImportableScrapingSession(@NotNull java.io.Reader reader)
reader
- A Reader
used to reader the characters.public void restoreHTTPParametersAfterScraping()
@Nullable @InternalOnly public java.lang.String reformatDate(java.lang.String date, java.lang.String dateFormatFrom, java.lang.String dateFormatTo) throws java.lang.Exception
date
- The date to be formatted, as a string.dateFormatFrom
- The pattern of the existing date.dateFormatTo
- The pattern the date should be converted to.java.text.ParseException
- If either of the date formats are invalid.java.lang.Exception
@Nullable @InternalOnly public java.lang.String reformatDate(java.lang.String date, java.lang.String dateFormatTo) throws java.lang.Exception
date
- The date to be formatted, as a string.dateFormatTo
- The pattern the date should be converted to.java.lang.Exception
public int getOriginatorEdition()
Settings
class.@InternalOnly public void setOriginatorEdition(int originatorEdition)
originatorEdition
- The edition, as defined by one of the constants in the Settings
class.@InternalOnly public void resizeImageFixWidth(@NotNull java.lang.String originalFile, @NotNull java.lang.String thumbnailFile, int newWidthSize, boolean deleteOriginalFile) throws java.lang.Exception
originalFile
- The path to the original file.thumbnailFile
- The path where the thumbnail should be created.newWidthSize
- The new width of the image.deleteOriginalFile
- Whether or not the original file should be deleted after the thumbnail is created.java.lang.Exception
@InternalOnly public void resizeImageFixHeight(@NotNull java.lang.String originalFile, @NotNull java.lang.String thumbnailFile, int newHeightSize, boolean deleteOriginalFile) throws java.lang.Exception
originalFile
- The path to the original file.thumbnailFile
- The path where the thumbnail should be created.newHeightSize
- The new height of the image.deleteOriginalFile
- Whether or not the original file should be deleted after the thumbnail is created.java.lang.Exception
@InternalOnly public void resizeImageFixWidthAndHeight(@NotNull java.lang.String originalFile, @NotNull java.lang.String thumbnailFile, int newWidthSize, int newHeightSize, boolean deleteOriginalFile) throws java.lang.Exception
originalFile
- The path to the original file.thumbnailFile
- The path where the thumbnail should be created.newWidthSize
- The new width of the image.newHeightSize
- The new height of the image.deleteOriginalFile
- Whether or not the original file should be deleted after the thumbnail is created.java.lang.Exception
@InternalOnly public int getImageHeight(@NotNull java.lang.String imagePath) throws java.lang.Exception
imagePath
- The path to the image.java.lang.Exception
@InternalOnly public int getImageWidth(@NotNull java.lang.String imagePath) throws java.lang.Exception
imagePath
- The path to the image.java.lang.Exception
@InternalOnly public int getMemoryUsage() throws java.lang.Exception
java.lang.Exception
public void logMemoryUsage()
public int getLoggingLevel()
public void setLoggingLevel(int loggingLevel)
loggingLevel
- The logging level.public void setLoggingLevelDebug()
public void setLoggingLevelInfo()
public void setLoggingLevelWarn()
public void setLoggingLevelError()
@InternalOnly public void setLoggingLevel(@NotNull java.lang.String loggingLevel)
Notifiable
class.loggingLevel
- A string.@InternalOnly public boolean getUsesNavigationActions()
@InternalOnly public void determineUsesNavigationActions()
@InternalOnly public void pushScriptContext(ScriptContext scriptContext)
ScriptContext
object on to the stack.pushScriptContext
in interface RunningScrapingSession
scriptContext
- A ScriptContext
object.@InternalOnly public void popScriptContext()
ScriptContext
object off the stack.popScriptContext
in interface RunningScrapingSession
public int getNumScriptsOnStack()
ScriptContext
objects on the stack.getNumScriptsOnStack
in interface RunningScrapingSession
@InternalOnly public int getMaxScriptsOnStack()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setMaxScriptsOnStack(int maxScriptsOnStack)
maxScriptsOnStack
- The number of scripts.@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getScrapeableSessionID()
@InternalOnly public void generateScrapeableSession()
@Nullable @InternalOnly public ScrapeableSession getScrapeableSession()
ScrapeableSession
if it's been created.protected void addChildScrapingSession(ScrapingSession childScrapingSession)
childScrapingSession
- The ScrapingSession
to add.protected void removeChildScrapingSession(ScrapingSession childScrapingSession)
childScrapingSession
- The ScrapingSession
to remove.@Nullable public ScrapingSession getParentScrapingSession()
ScrapingSession
.protected void setParentScrapingSession(ScrapingSession parentScrapingSession)
parentScrapingSession
- A ScrapingSession
.public void stopChildScrapingSessions()
@Nullable public java.lang.String getCharacterSet()
public void setCharacterSet(@Nullable java.lang.String characterSet)
characterSet
- The character set.@InternalOnly public void setLastDeterminedCharacterSet(@Nullable java.lang.String characterSet)
characterSet
- The last determined character set from a server, or null if none is known@Nullable public java.lang.String getLastDeterminedCharacterSet()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public boolean getUseServerCharacterSet()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setUseServerCharacterSet(boolean useServerCharacterSet)
useServerCharacterSet
- A boolean.@InternalOnly public static java.lang.String getCharacterSetFromScrapingSessionExportFile(@NotNull java.io.File file, @NotNull java.lang.String charSet)
file
- The exported scraping session file.charSet
- The charset to try when reading in the file.@InternalOnly public static java.lang.String getCharacterSetFromScrapingSessionExportFile(@NotNull java.io.File file)
file
- The exported scraping session file.@Nullable public java.lang.String getDateExported()
@InternalOnly public void setDateExported(java.lang.String date)
date
- The date as a string.public long getDateUpdated()
@InternalOnly public void setDateUpdated(long dateUpdated)
dateUpdated
- A long.@InternalOnly public boolean getNeedsToBeSaved()
@InternalOnly public void setNeedsToBeSaved(boolean needsToBeSaved)
needsToBeSaved
- A boolean.public int getConnectionTimeout()
public void setConnectionTimeout(int connectionTimeout)
connectionTimeout
- The connection timeout in seconds.@Nullable public java.lang.String getKeyStoreFilePath()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setKeyStoreFilePath(java.lang.String keyStoreFilePath)
keyStoreFilePath
- The path to the keystore file.@Nullable public java.lang.String getKeyStorePassword()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setKeyStorePassword(java.lang.String keyStorePassword)
public boolean getRandomizeUserAgent()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setRandomizeUserAgent(boolean randomizeUserAgent)
randomizeUserAgent
- A boolean.@Nullable public java.lang.String getUserAgent()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setUserAgent(@Nullable java.lang.String userAgent)
userAgent
- The user-agent.@NotNull public java.lang.String getDefaultUserAgent()
public void setDefaultUserAgent(java.lang.String defaultUserAgent)
defaultUserAgent
- The default user agent.@NotNull @InternalOnly public java.lang.Object[] getDataToSendToClientAsArrayAndFlush()
HashMap
objects, then flushes out all of the data from the array list.@RequiredVersion(value=2) public void sendDataToClient(@Nullable java.lang.String key, @Nullable java.lang.Object value)
sendDataToClient
in interface RunningScrapingSession
key
- The key identifier of the data.value
- The value portion of the data.@InternalOnly public boolean isDataFlavorSupported(@NotNull java.awt.datatransfer.DataFlavor df)
isDataFlavorSupported
in interface java.awt.datatransfer.Transferable
@NotNull @InternalOnly public java.lang.Object getTransferData(@NotNull java.awt.datatransfer.DataFlavor df) throws java.awt.datatransfer.UnsupportedFlavorException
getTransferData
in interface java.awt.datatransfer.Transferable
java.awt.datatransfer.UnsupportedFlavorException
@NotNull @InternalOnly public java.awt.datatransfer.DataFlavor[] getTransferDataFlavors()
getTransferDataFlavors
in interface java.awt.datatransfer.Transferable
@InternalOnly public void dumpToLog()
public void setTimeZone(@Nullable java.lang.String timeZone)
timeZone
- The new timezone to use. If null is given, the local timezone will be used.public void setTimeZone(@Nullable java.util.TimeZone timeZone)
timeZone
- The new timezone to use. If null is given, the local timezone will be used.@Nullable public java.util.TimeZone getTimeZone()
public boolean isRunningInWorkbench()
public boolean isRunningFromCommandLine()
public boolean isRunningInServer()
public boolean getDebugMode()
public void setDebugMode(boolean debugMode)
debugMode
- True to enable debug mode.public void setVariables(@NotNull java.util.Map<java.lang.Object,java.lang.Object> variables)
variables
- The map to use when setting the session variables@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setVariables(@NotNull java.util.Map<java.lang.Object,java.lang.Object> variables, boolean ignoreLowerCaseKeys)
variables
- The map to use when setting the session variablesignoreLowerCaseKeys
- True if keys with lowercase characters should be ignored@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setSessionVariablesFromMap(@NotNull java.util.Map<java.lang.Object,java.lang.Object> map, boolean ignoreLowerCaseKeys)
map
- The map to use when setting the session variablesignoreLowerCaseKeys
- True if keys with lowercase characters should be ignoredpublic void clearVariables(@NotNull java.util.Map<java.lang.Object,java.lang.Object> variables)
variables
- The variables to clear in the sessionpublic void clearVariables(@NotNull java.util.Collection<java.lang.Object> variables)
variables
- The variables to clear in the session@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void removeSessionVariablesInMap(@NotNull java.util.Map<java.lang.Object,java.lang.Object> variables, boolean ignoreLowerCaseKeys)
variables
- The map to use when clearing the keys from the sessionignoreLowerCaseKeys
- True if keys with lowercase characters should be ignoredpublic java.util.Date getLastImportTime()
@NotNull @InternalOnly public ScrapeStateLogger getScrapeLogger()
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.Object addMonitoredValue(java.lang.String name, java.lang.Object value)
name
- The name for the value being monitoredvalue
- The value to associate with the given name@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.Object removeMonitoredValue(java.lang.String name)
name
- The name for the value to remove@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addMonitoredVariable(java.lang.String key)
key
- The key in the session corresponding to a value@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addMonitoredPrefix(java.lang.String prefix)
prefix
- The prefix to monitor@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addMonitoredPostfix(java.lang.String postfix)
postfix
- The postfix to monitor@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void removeMonitoredVariable(java.lang.String key)
key
- Key for the variable to stop watching@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void removeMonitoredPrefix(java.lang.String prefix)
prefix
- Prefix to remove from monitoring@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void removeMonitoredPostfix(java.lang.String postfix)
postfix
- Postfix to remove from monitoringpublic void logMonitoredValues(java.lang.Object message) throws java.lang.Exception
message
- A message to output as a header for this log entryjava.lang.Exception
@InternalOnly public void logMonitoredValues(java.lang.Object message, int logLevel)
message
- A message to output as a header for this log entrylogLevel
- The level to log at@RequiredVersion(value=1) public void logMonitoredValuesDebug(java.lang.Object message)
message
- A message to output as a header for this log entry@RequiredVersion(value=1) public void logMonitoredValuesInfo(java.lang.Object message)
message
- A message to output as a header for this log entry@RequiredVersion(value=1) public void logMonitoredValuesWarn(java.lang.Object message)
message
- A message to output as a header for this log entry@RequiredVersion(value=1) public void logMonitoredValuesError(java.lang.Object message)
message
- A message to output as a header for this log entry@RequiredVersion(value=1) public void logMonitoredValuesClose(java.lang.Object message)
message
- The message to display as a headerpublic void logDataRecord(DataRecord record)
record
- The DataRecord to write to the log@InternalOnly public void logDataRecord(DataRecord record, int logLevel)
record
- The DataRecord to write to the loglogLevel
- The level to log the data record at, as an int@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logDataRecordDebug(DataRecord record)
record
- The DataRecord to log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logDataRecordInfo(DataRecord record)
record
- The DataRecord to log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logDataRecordWarn(DataRecord record)
record
- The DataRecord to log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logDataRecordError(DataRecord record)
record
- The DataRecord to logpublic void logException(java.lang.Throwable exception)
exception
- The Exception to logpublic void logObjectByType(java.lang.Object object)
object
- The Object to write to the log@InternalOnly public void logObjectByType(java.lang.Object object, int logLevel)
object
- The Object to write to the loglogLevel
- The level to log the data record at, as an int@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logObjectByTypeDebug(java.lang.Object object)
object
- The Object to write to the log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logObjectByTypeInfo(java.lang.Object object)
object
- The Object to write to the log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logObjectByTypeWarn(java.lang.Object object)
object
- The Object to write to the log@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void logObjectByTypeError(java.lang.Object object)
object
- The Object to write to the log@RequiredVersion(value=1) public void webDebug(java.lang.Object message)
message
- The message to display as a header@RequiredVersion(value=1) public void webDebug(java.lang.Object message, java.lang.Object loggable)
message
- The message to display as a headerloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webDebug(java.lang.Object message, boolean saveMessage)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messages@RequiredVersion(value=1) public void webDebug(java.lang.Object message, boolean saveMessage, @Nullable java.lang.Object loggable)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messagesloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webInfo(java.lang.Object message)
message
- The message to display as a header@RequiredVersion(value=1) public void webInfo(java.lang.Object message, java.lang.Object loggable)
message
- The message to display as a headerloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webInfo(java.lang.Object message, boolean saveMessage)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messages@RequiredVersion(value=1) public void webInfo(java.lang.Object message, boolean saveMessage, @Nullable java.lang.Object loggable)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messagesloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webWarn(java.lang.Object message)
message
- The message to display as a header@RequiredVersion(value=1) public void webWarn(java.lang.Object message, java.lang.Object loggable)
message
- The message to display as a headerloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webWarn(java.lang.Object message, boolean saveMessage)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messages@RequiredVersion(value=1) public void webWarn(java.lang.Object message, boolean saveMessage, @Nullable java.lang.Object loggable)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messagesloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webError(java.lang.Object message)
message
- The message to display as a header@RequiredVersion(value=1) public void webError(java.lang.Object message, java.lang.Object loggable)
message
- The message to display as a headerloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webError(java.lang.Object message, boolean saveMessage)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messages@RequiredVersion(value=1) public void webError(java.lang.Object message, boolean saveMessage, @Nullable java.lang.Object loggable)
message
- The message to display as a headersaveMessage
- Whether or not to save this message and continue to display it below future messagesloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other values@RequiredVersion(value=1) public void webClose(java.lang.Object message)
message
- The message to display as a header@RequiredVersion(value=1) public void webClose(java.lang.Object message, @Nullable java.lang.Object loggable)
message
- The message to display as a headerloggable
- An additional object to log, most likely a DataRecord. This will only be logged with this message, and not 'monitored' like other valuespublic void logScreenScraperInformation()
@NotNull @RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar addProgressBar(java.lang.String title)
title
- The title for the new progress bar@NotNull @RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar addProgressBar(java.lang.String title, @NotNull java.lang.String total)
title
- The title for the new progress bartotal
- The total for the new progress barjava.lang.NumberFormatException
- total isn't parseable as a Double@NotNull @RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar addProgressBar(java.lang.String title, double total)
title
- The title for the new progress bartotal
- The total for the new progress bar@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void removeProgressBar(ProgressBar progressBar)
progressBar
- The ProgressBar to remove@Nullable @RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar getProgressBar(int index)
index
- The desired ProgressBar's index@Nullable @RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar getProgressBar(java.lang.String title)
title
- The title to search for@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public int getNumProgressBars()
@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public ProgressBar getLastProgressBar()
void unlockBreakpointBlockedThreads()
void resetThreadGate()
void resetCallbacks()
public void waitForBreakpoint()
@Deprecated @InternalOnly public void waitForBreakpoint(boolean pauseBackgroundThreads)
pauseBackgroundThreads
- Should background threads calling this method continue running anyway.@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) @Nullable public FileCacher getScrapeCache()
@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) @Nullable public FileCacher getUsableScrapeCache()
void setScrapeCacher(@Nullable FileCacher cacher)
cacher
- The new cacher to use. Null values do not clear a cacher if one already exists.@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void enableCaching(java.lang.String description, boolean saveLogs, boolean zipCachedFiles)
description
- A description to use in the cached file namesaveLogs
- True if logs should be included in the cachezipCachedFiles
- True if the cached files should be zipped once the scrape ends.@InternalOnly public void endCachingAtScrapeClose() throws java.io.IOException
java.io.IOException
- If there was an error writing the zip or renaming the cache directory@Nullable @RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.io.File endCaching() throws java.io.IOException
java.io.IOException
- If there was an error writing the zip or renaming the cache directorypublic boolean getCachingEnabled()
@Nullable public java.io.File cacheBytes(java.lang.String outputFilenameAndPath, byte[] bytes)
outputFilenameAndPath
- The name of the file in the cache, including any directory it should be placed inbytes
- The content of the file that should be cached.java.io.IOException
- If there is an error writing the bytes to the cache directory for the scrape@Nullable public java.io.File cacheFile(java.lang.String outputFilenameAndPath, java.io.File fileToCache)
outputFilenameAndPath
- The name of the file in the cache, including any directory it should be placed infileToCache
- The file that should be cached. This cannot be a directoryjava.io.IOException
- If there is an error copying the file to the new location.@Nullable public java.io.File cacheText(java.lang.String name, java.lang.String content) throws java.io.IOException
name
- The name for the file containing the cached text. This can include a pathcontent
- The content to place in the cachejava.io.IOException
- When there is a problem caching the text to a file@Nullable public java.io.File cacheText(java.lang.String name, java.lang.String content, @Nullable java.lang.String encoding)
name
- The name for the file containing the cached text. This can include a pathcontent
- The content to place in the cacheencoding
- The encoding to use for the text, or null to use the default encoding for Javajava.io.IOException
- When there is a problem caching the text to a file@Nullable public java.io.File cacheScrapeableFile(@NotNull ScrapeableFile scrapeableFile) throws java.io.IOException
scrapeableFile
- The file to cachejava.io.IOException
- When failure occurs writing the file to disk@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addAutoProgressBar(java.lang.String name, java.lang.String... values)
name
- The name of the progress bar, which should match the session variable where the value for updating this bar will be storedvalues
- The values this progress bar can have, in the order they will be queried@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addAutoProgressBar(java.lang.String name, @NotNull java.lang.String[][] values)
name
- The name of the progress bar, which should match the session variable where the value for updating this bar will be storedvalues
- The values this progress bar can have, in the order they will be queried@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addAutoProgressBar(java.lang.String name, @NotNull java.lang.String[][] values, int keyIndex)
name
- The name of the progress bar, which should match the session variable where the value for updating this bar will be storedvalues
- The values this progress bar can have, in the order they will be queriedkeyIndex
- The index in each inner array of the value that will be set in the session variable@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addAutoProgressBar(java.lang.String name, @NotNull java.util.Collection<java.lang.String> values)
name
- The name of the progress bar, which should match the session variable where the value for updating this bar will be storedvalues
- The values this progress bar can have, in the order they will be queried@RequiredVersion(value=2, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void addAutoProgressBar(java.lang.String name, @NotNull DataSet values, @NotNull java.lang.String key)
name
- The name of the progress bar, which should match the session variable where the value for updating this bar will be storedvalues
- The values this progress bar can have, in the order they will be queriedkey
- The key in the DataRecords that will be used for the session variable matching name@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void setDefaultRetryPolicy(@Nullable RetryPolicy retryPolicy)
retryPolicy
- The retry policy to use by default, if no other retry policy is set.@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) @Nullable public RetryPolicy getDefaultRetryPolicy()
@RequiredVersion(value=1) public void executeScriptWithContext(java.lang.String scriptName) throws java.lang.Exception
scriptName
- The name of the script to execute.java.lang.Exception
- if the script couldn't be found@InternalOnly public void handleScriptError(@NotNull ScriptException scriptException, ScriptContext scriptContext)
scriptException
- The ScriptException
that was thrown.scriptContext
- The ScriptContext
that produced the error.public java.lang.String saveStateToString(boolean saveCookies, boolean saveVariables)
saveCookies
- Should cookies be savedsaveVariables
- Should variables be saved@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public java.lang.String saveStateToString(boolean saveCookies, @Nullable java.util.Collection<java.lang.String> saveVariables)
saveCookies
- Should cookies be savedsaveVariables
- A Collection of variables to save, or null to save all of them@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void loadStateFromString(@Nullable java.lang.String stateXML)
stateXML
- The xml containing the state to loadpublic void breakpoint(boolean logVariablesInNonWorkbench)
logVariablesInNonWorkbench
- True if session variables should be written to the log when not running in workbench modejava.lang.Exception
- In editions that are not allowed to call this methodpublic void breakpoint(@Nullable java.lang.Object message)
message
- A message to displayjava.lang.Exception
- In editions that are not allowed to call this method@RequiredVersion(value=1, behavior=LOG_ERROR_AND_RETURN_NULL_NEG_ONE_OR_FALSE) public void breakpoint(@Nullable java.lang.Object message, boolean logVariablesInNonWorkbench)
message
- A message to displaylogVariablesInNonWorkbench
- True if session variables should be written to the log when not running in workbench mode@InternalOnly @NotNull public java.lang.String[] getCurrentStack()
@NotNull public java.util.Map<java.lang.String,java.lang.Object> getAllVariables()
@NotNull @InternalOnly public java.util.Map<java.lang.String,java.lang.Object> getAllInternalVariables()
@RequiredVersion(value=1) public boolean getStopScrapingOnScriptError()
@RequiredVersion(value=1) public void setStopScrapingOnScriptError(boolean stopScrapingOnScriptError)
stopScrapingOnScriptError
- A boolean.@RequiredVersion(value=1) public boolean getStopScrapingOnMaxRequestAttemptsReached()
@RequiredVersion(value=1) public void setStopScrapingOnMaxRequestAttemptsReached(boolean stopScrapingOnMaxRequestAttemptsReached)
stopScrapingOnMaxRequestAttemptsReached
- A boolean.@RequiredVersion(value=1) public boolean getStopScrapingOnExtractorPatternTimeout()
@RequiredVersion(value=1) public void setStopScrapingOnExtractorPatternTimeout(boolean stopScrapingOnExtractorPatternTimeout)
stopScrapingOnExtractorPatternTimeout
- A boolean.public boolean getUseOnlySSLv3()
public void setUseOnlySSLv3(boolean useOnlySSLv3)
useOnlySSLv3
- A boolean.public void clearProxySettings()
public void addRandomDelayBetweenRequests(long minMs, long maxMs)
minMs
- The minimum number of milliseconds to waitmaxMs
- The maximum number of milliseconds to wait@NotNull @RequiredVersion(value=1) public EventEngine getEventEngine()
@NotNull @InternalOnly public EventEngine getEventEngineInternal()
@Nullable @InternalOnly public java.lang.String getIdentifier()
@InternalOnly public void setIdentifier(java.lang.String identifier)
public void setAutoCloseAfterScrapeEnds(@NotNull java.io.Closeable closeable)
closeable
- The closeable to close on completion of the session