Serialized Form


Package com.norconex.collector.http

Class com.norconex.collector.http.HttpCollectorConfig extends Object implements Serializable

serialVersionUID: -3350877963428801802L

Serialized Fields

id

String id

crawlerConfigs

HttpCrawlerConfig[] crawlerConfigs

progressDir

String progressDir

logsDir

String logsDir

Class com.norconex.collector.http.HttpCollectorException extends RuntimeException implements Serializable

serialVersionUID: -805913995358009121L


Package com.norconex.collector.http.crawler

Class com.norconex.collector.http.crawler.CrawlURL extends Object implements Serializable

serialVersionUID: -2219206220476107409L

Serialized Fields

depth

int depth

url

String url

status

CrawlStatus status

headChecksum

String headChecksum

docChecksum

String docChecksum

Class com.norconex.collector.http.crawler.HttpCrawlerConfig extends Object implements Serializable

serialVersionUID: -3350877963428801802L

Serialized Fields

id

String id

maxDepth

int maxDepth

workDir

File workDir

startURLs

String[] startURLs

numThreads

int numThreads

maxURLs

int maxURLs

ignoreRobotsTxt

boolean ignoreRobotsTxt

keepDownloads

boolean keepDownloads

deleteOrphans

boolean deleteOrphans

urlNormalizer

IURLNormalizer urlNormalizer

delayResolver

IDelayResolver delayResolver

httpClientInitializer

IHttpClientInitializer httpClientInitializer

httpDocumentFetcher

IHttpDocumentFetcher httpDocumentFetcher

httpHeadersFetcher

IHttpHeadersFetcher httpHeadersFetcher

urlExtractor

IURLExtractor urlExtractor

robotsTxtProvider

IRobotsTxtProvider robotsTxtProvider

crawlURLDatabaseFactory

ICrawlURLDatabaseFactory crawlURLDatabaseFactory

importerConfig

ImporterConfig importerConfig

documentfilters

IHttpDocumentFilter[] documentfilters

urlFilters

IURLFilter[] urlFilters

httpHeadersFilters

IHttpHeadersFilter[] httpHeadersFilters

httpHeadersChecksummer

IHttpHeadersChecksummer httpHeadersChecksummer

preImportProcessors

IHttpDocumentProcessor[] preImportProcessors

postImportProcessors

IHttpDocumentProcessor[] postImportProcessors

httpDocumentChecksummer

IHttpDocumentChecksummer httpDocumentChecksummer

crawlerListeners

IHttpCrawlerEventListener[] crawlerListeners

committer

ICommitter committer

Package com.norconex.collector.http.db

Class com.norconex.collector.http.db.CrawlURLDatabaseException extends RuntimeException implements Serializable

serialVersionUID: 5416591514078326431L


Package com.norconex.collector.http.db.impl

Class com.norconex.collector.http.db.impl.DefaultCrawlURLDatabaseFactory extends Object implements Serializable

serialVersionUID: 6088230386061613319L


Package com.norconex.collector.http.doc

Class com.norconex.collector.http.doc.HttpDocument extends Object implements Serializable

serialVersionUID: 4376740210800410675L

Serialized Fields

url

String url

localFile

File localFile

metadata

HttpMetadata metadata

Class com.norconex.collector.http.doc.HttpMetadata extends Properties implements Serializable

serialVersionUID: 1454870639551983430L


Package com.norconex.collector.http.filter

Package com.norconex.collector.http.filter.impl

Class com.norconex.collector.http.filter.impl.ExtensionURLFilter extends AbstractOnMatchFilter implements Serializable

serialVersionUID: -8029862304058855686L

Serialized Fields

caseSensitive

boolean caseSensitive

extensions

String extensions

extensionParts

String[] extensionParts

Class com.norconex.collector.http.filter.impl.RegexHeaderFilter extends AbstractOnMatchFilter implements Serializable

serialVersionUID: -8029862304058855686L

Serialized Fields

caseSensitive

boolean caseSensitive

header

String header

regex

String regex

pattern

Pattern pattern

Class com.norconex.collector.http.filter.impl.RegexURLFilter extends AbstractOnMatchFilter implements Serializable

serialVersionUID: -8029862304058855686L

Serialized Fields

caseSensitive

boolean caseSensitive

regex

String regex

pattern

Pattern pattern

Package com.norconex.collector.http.handler

Package com.norconex.collector.http.handler.impl

Class com.norconex.collector.http.handler.impl.DefaultDelayResolver extends Object implements Serializable

serialVersionUID: -7742290966880042419L

Serialized Fields

lastHitTimestampNanos

long lastHitTimestampNanos

defaultDelay

long defaultDelay

schedules

List<E> schedules

ignoreRobotsCrawlDelay

boolean ignoreRobotsCrawlDelay

Class com.norconex.collector.http.handler.impl.DefaultDelayResolver.DelaySchedule extends Object implements Serializable

serialVersionUID: 5602602696446583844L

Serialized Fields

dayOfWeekRange

Range<T> dayOfWeekRange

dayOfMonthRange

Range<T> dayOfMonthRange

timeRange

ImmutablePair<L,R> timeRange

delay

long delay

Class com.norconex.collector.http.handler.impl.DefaultDocumentFetcher extends Object implements Serializable

serialVersionUID: -6523482835344340418L

Serialized Fields

validStatusCodes

int[] validStatusCodes

headersPrefix

String headersPrefix

Class com.norconex.collector.http.handler.impl.DefaultHttpClientInitializer extends Object implements Serializable

serialVersionUID: 8489434479618081974L

Serialized Fields

authMethod

String authMethod

authURL

String authURL

authUsernameField

String authUsernameField

authUsername

String authUsername

authPasswordField

String authPasswordField

authPassword

String authPassword

cookiesDisabled

boolean cookiesDisabled

userAgent

String userAgent

proxyHost

String proxyHost

proxyPort

int proxyPort

proxyUsername

String proxyUsername

proxyPassword

String proxyPassword

proxyRealm

String proxyRealm

Class com.norconex.collector.http.handler.impl.DefaultHttpDocumentChecksummer extends Object implements Serializable

serialVersionUID: 3795335571186097378L

Serialized Fields

field

String field

Class com.norconex.collector.http.handler.impl.DefaultHttpHeadersChecksummer extends Object implements Serializable

serialVersionUID: -6759418012119786557L

Serialized Fields

field

String field

Class com.norconex.collector.http.handler.impl.DefaultRobotsTxtProvider extends Object implements Serializable

serialVersionUID: 1459917072724725590L

Serialized Fields

robotsTxtCache

Map<K,V> robotsTxtCache

Class com.norconex.collector.http.handler.impl.DefaultURLExtractor extends Object implements Serializable

serialVersionUID: 4130729871145622411L

Serialized Fields

maxURLLength

int maxURLLength

Class com.norconex.collector.http.handler.impl.GenericURLNormalizer extends Object implements Serializable

serialVersionUID: 3847717776792688065L

Serialized Fields

normalizations

List<E> normalizations

replaces

List<E> replaces

Class com.norconex.collector.http.handler.impl.SimpleHttpHeadersFetcher extends Object implements Serializable

serialVersionUID: 6526443843689019304L

Serialized Fields

validStatusCodes

int[] validStatusCodes

headersPrefix

String headersPrefix

Package com.norconex.collector.http.robot

Class com.norconex.collector.http.robot.RobotsTxt extends Object implements Serializable

serialVersionUID: -2203572498193869416L

Serialized Fields

filters

IURLFilter[] filters

crawlDelay

float crawlDelay



Copyright © 2009-2013 Norconex Inc.. All Rights Reserved.