|
|
@@ -1,25 +1,28 @@
|
|
|
+import threading
|
|
|
from concurrent.futures import ThreadPoolExecutor, wait
|
|
|
|
|
|
-from crawler.spiders import SearchEngine, VisitDomain
|
|
|
+from crawler.spiders import SearchEngine, VisitDomain, SyncData
|
|
|
from crawler.utils import err_details
|
|
|
|
|
|
|
|
|
class BreadthCrawler(SearchEngine, VisitDomain):
|
|
|
|
|
|
- def __init__(self, workers=2, **kwargs):
|
|
|
+ def __init__(self, workers=1, **kwargs):
|
|
|
+ SyncData(**kwargs)
|
|
|
SearchEngine.__init__(self, **kwargs)
|
|
|
VisitDomain.__init__(self, **kwargs)
|
|
|
- self._workers = workers if workers < 2 else 2
|
|
|
+ self._workers = workers
|
|
|
|
|
|
def start(self):
|
|
|
+ threading.Thread(
|
|
|
+ target=self.load_engines,
|
|
|
+ name='MainSearchEngine'
|
|
|
+ ).start()
|
|
|
with ThreadPoolExecutor(max_workers=self._workers) as executor:
|
|
|
futures = []
|
|
|
- f_engine = executor.submit(self.search_engines)
|
|
|
- f_engine.add_done_callback(err_details)
|
|
|
- futures.append(f_engine)
|
|
|
for _ in range(1, self._workers + 1):
|
|
|
- f_domain = executor.submit(self.search_domains)
|
|
|
- f_domain.add_done_callback(err_details)
|
|
|
- futures.append(f_domain)
|
|
|
+ f = executor.submit(self.search_domains)
|
|
|
+ f.add_done_callback(err_details)
|
|
|
+ futures.append(f)
|
|
|
wait(futures)
|
|
|
- print('寻源任务结束')
|
|
|
+ print('寻源任务结束')
|
