Index

A | C | D | E | F | G | H | I | M | P | Q | R | S | U | Z

A

AbstractBaseFrontier (class in spyder.core.frontier)
AbstractCrawlUriSink (class in spyder.core.sink)
add_or_create_queue() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
add_sink() (spyder.core.frontier.AbstractBaseFrontier method)
add_uri() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
add_uris() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
all_uris() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
AsyncZmqWorker (class in spyder.core.worker)

C

checkpoint() (spyder.core.sqlitequeues.SQLiteStore method)
close() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.master.ZmqMaster method)
(spyder.core.sqlitequeues.SQLiteStore method)
(spyder.core.worker.ZmqWorker method)
copy_skeleton_dir() (in module spyder)
create_frontier() (in module spyder.masterprocess)
create_master_management() (in module spyder.masterprocess)
create_processing_function() (in module spyder.workerprocess)
create_worker_extractor() (in module spyder.workerprocess)
create_worker_fetcher() (in module spyder.workerprocess)
create_worker_management() (in module spyder.workerprocess)

D

DefaultHtmlLinkExtractor (class in spyder.processor.htmllinkextractor)

E

extract_info_from_response() (in module spyder.processor.fetcher)

F

FetchProcessor (class in spyder.processor.fetcher)
finished() (spyder.core.master.ZmqMaster method)

G

get_all_queues() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
get_next() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.frontier.MultipleHostFrontier method)
(spyder.core.frontier.SingleHostFrontier method)
get_queue_count() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
get_queue_for_ident() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
get_uri() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)

H

handle_response() (in module spyder.processor.fetcher)

I

ignore_uri() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)

M

main() (in module spyder.masterprocess)
(in module spyder.workerprocess)
MultipleHostFrontier (class in spyder.core.frontier)

P

prepare_headers() (in module spyder.processor.fetcher)
process_not_found() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.frontier.MultipleHostFrontier method)
(spyder.core.sink.AbstractCrawlUriSink method)
process_redirect() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.frontier.MultipleHostFrontier method)
(spyder.core.sink.AbstractCrawlUriSink method)
process_server_error() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.frontier.MultipleHostFrontier method)
(spyder.core.sink.AbstractCrawlUriSink method)
process_successful_crawl() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.frontier.MultipleHostFrontier method)
(spyder.core.frontier.SingleHostFrontier method)
(spyder.core.sink.AbstractCrawlUriSink method)

Q

qsize() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
queue_head() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
QueueException
QueueNotFound

R

RegexScoper (class in spyder.processor.scoper)
remove_uris() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)

S

shutdown() (spyder.core.master.ZmqMaster method)
SingleHostFrontier (class in spyder.core.frontier)
spyder (module)
spyder.core.frontier (module)
spyder.core.master (module)
spyder.core.sink (module)
spyder.core.sqlitequeues (module)
spyder.core.worker (module)
spyder.masterprocess (module)
spyder.processor.fetcher (module)
spyder.processor.htmllinkextractor (module)
spyder.processor.scoper (module)
spyder.workerprocess (module)
spyder_admin_main() (in module spyder)
spyder_management() (in module spyder)
SQLiteMultipleHostUriQueue (class in spyder.core.sqlitequeues)
SQLiteSingleHostUriQueue (class in spyder.core.sqlitequeues)
SQLiteStore (class in spyder.core.sqlitequeues)
start() (spyder.core.master.ZmqMaster method)
(spyder.core.worker.ZmqWorker method)
stop() (spyder.core.master.ZmqMaster method)
(spyder.core.worker.ZmqWorker method)

U

update_uri() (spyder.core.frontier.AbstractBaseFrontier method)
(spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
update_uris() (spyder.core.sqlitequeues.SQLiteMultipleHostUriQueue method)
(spyder.core.sqlitequeues.SQLiteSingleHostUriQueue method)
UriNotFound

Z

ZmqMaster (class in spyder.core.master)
ZmqWorker (class in spyder.core.worker)