代码更新6.16

a_bjgc_bggs/变更公示-详情页.py

@@ -133,7 +133,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies(), headers=headers)
+                        proxies=request.get_proxies(), headers=headers)
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_bjgc_jggs/结果公示-详情页.py

@@ -71,7 +71,7 @@ class Details(feapder.BiddingDetailSpider):
                 file_type = extract_file_type(file_name,file_url)
                 attachment = AttachmentDownloader().fetch_attachment(
                     file_name=file_name, file_type=file_type, download_url=file_url,
-                    proxies=request.proxies(),headers=headers)
+                    proxies=request.get_proxies(),headers=headers)
                 attachments[str(len(attachments) + 1)] = attachment
 
                 fhtml = f'''

a_gdsggzyjypt_bcgdzb_jggg/补充耕地指标-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_bcgdzb_jygg/补充耕地指标-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_bcgdzb_jygg/补充耕地指标-交易公告-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_fkzbjggg/其他交易-复垦指标结果公告-列表页.py

@@ -88,7 +88,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_fkzbjygg/其他交易-复垦指标交易公告-列表页.py

@@ -88,7 +88,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_gycq_jggg/国有产权-结果公告-列表页.py

@@ -88,7 +88,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_hlzy_jggg/河流资源-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_hlzy_jggg/河流资源-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_hlzy_jygg/河流资源-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_hlzy_jygg/河流资源-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_hyzy_jggg/海洋资源-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_hyzy_jggg/海洋资源-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_hyzy_jygg/海洋资源-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_kysyq_jggg/空域使用权-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_kysyq_jggg/空域使用权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_kysyq_jygg/空域使用权-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_kysyq_jygg/空域使用权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_lqjy_xxpl/林权交易-信息披露-列表页.py

@@ -88,7 +88,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_pwq_jggg/排污权-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_pwq_jggg/排污权-详情页.py

@@ -112,7 +112,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -134,7 +134,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -146,7 +146,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_pwq_jygg/排污权-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_pwq_jygg/排污权-详情页.py

@@ -112,7 +112,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -134,7 +134,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -146,7 +146,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_sfsszc_jggg/涉法涉诉资产-结果公告-列表页.py

@@ -88,7 +88,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_sfsszc_jggg/涉法涉诉资产-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_sfsszc_jygg/涉法涉诉资产-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tdky_kyqjggg/土地矿业-矿业权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_tdky_kyqjggg/土地矿业-矿业权交易结果公告-列表页.py

@@ -91,7 +91,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tdky_kyqjygg/土地矿业-矿业权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_tdky_kyqjygg/土地矿业-矿业权交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tdky_tdsyqjggg/土地矿业-土地使用权交易结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tpfq_csgg/碳排放权-出售公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tpfq_csgg/碳排放权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_tpfq_csjggg/碳排放权-出售结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_tpfq_csjggg/碳排放权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_wxzc_jggg/无形资产-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_wxzc_jggg/无形资产-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_wxzc_jygg/无形资产-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_wxzc_jygg/无形资产-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_ynq_jggg/用能权-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_ynq_jggg/用能权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_ynq_jygg/用能权-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_ynq_jygg/用能权-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gdsggzyjypt_yphyyhc_jggg/药品和医用耗材-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zfcg_cgjh/政府采购-采购计划-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zfcg_cgxq/政府采购-采购需求-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zfcg_cgyxgk/政府采购-采购意向公开-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zxkczlzbjj_jggg/中小客车增量指标竞价-结果公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zxkczlzbjj_jygg/中小客车增量指标竞价-交易公告-列表页.py

@@ -89,7 +89,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)

a_gdsggzyjypt_zxkczlzbjj_jygg/中小客车增量指标竞价-详情页.py

@@ -115,7 +115,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -137,7 +137,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,7 +149,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:

a_gjdtdzswpt_cgxmxx_bgecgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,15 +24,7 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=50)
@@ -42,43 +32,41 @@ class Details(feapder.BiddingDetailSpider):
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
-        if file_url_org:
-            file_url_re = re.findall('file=(.*?)&',file_url_org)
+        if file_url_org is not None:
+            file_url_re = re.findall('file=(.*?)&', file_url_org)
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title,
+                    file_type='pdf',
+                    download_url=file_url,
+                    cookies=cookies,
+                    headers=headers,
+                    proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_cgxmxx_zbgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,15 +24,7 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=50)
@@ -42,43 +32,37 @@ class Details(feapder.BiddingDetailSpider):
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
-        if file_url_org:
+        if file_url_org is not None:
             file_url_re = re.findall('file=(.*?)&',file_url_org)
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_cgxmxx_zbhxrgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,15 +24,7 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=50)
@@ -42,22 +32,19 @@ class Details(feapder.BiddingDetailSpider):
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_cgxmxx_zbjggg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,15 +24,7 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=50)
@@ -42,22 +32,19 @@ class Details(feapder.BiddingDetailSpider):
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_cgxmxx_zzgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,15 +24,7 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=50)
@@ -42,22 +32,19 @@ class Details(feapder.BiddingDetailSpider):
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_fzxx_bgecgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,38 +24,27 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        data_list = self.get_tasks_by_rabbitmq(limit=50)
+        data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies)
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_fzxx_cggg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,38 +24,27 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        data_list = self.get_tasks_by_rabbitmq(limit=50)
+        data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_fzxx_zbjggg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,38 +24,27 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        data_list = self.get_tasks_by_rabbitmq(limit=50)
+        data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_fzxx_zzgg/国家电投电子商务平台-详情页.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-03-12
+Created on 2025-05-29
 ---------
 @summary: 国家电投电子商务平台
 ---------
 @author: lzz
 """
+import re
+from urllib import parse
+
 import feapder
 from items.spider_item import DataBakItem
-from feapder.utils.tools import log
 from untils.attachment import AttachmentDownloader
-from urllib import parse
-import re
-
-
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -26,38 +24,27 @@ headers = {
 }
 
 
-class Details(feapder.BiddingDetailSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        data_list = self.get_tasks_by_rabbitmq(limit=50)
+        data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             # log.debug(item)
             request_params = item.get("request_params")
             yield feapder.Request(url=item.get("parse_url"), item=item,render=True,
-                                  deal_detail=item.get("deal_detail"),render_time=5,
+                                  deal_detail=item.get("deal_detail"),render_time=10,
                                   callback=eval(item.get("parse")),  **request_params)
 
-
     def download_midware(self, request):
         request.headers = headers
 
-
     def detail_get(self, request, response):
-
         driver = response.browser
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         cookies = driver.cookies
-        list_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
+        data_item.contenthtml = response.xpath('//div[@class="textbox"]').extract_first() or ""
 
         attachments = {}
         file_url_org = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
@@ -66,19 +53,16 @@ class Details(feapder.BiddingDetailSpider):
             if file_url_re:
                 file_url = parse.unquote(file_url_re[0])
                 attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type='pdf', download_url=file_url,
-                    cookies=cookies,headers=headers,proxies=request.proxies())
+                    file_name=data_item.title, file_type='pdf', download_url=file_url,
+                    cookies=cookies,headers=headers,proxies=request.get_proxies())
                 attachments[str(len(attachments) + 1)] = attachment
-                list_item.contenthtml = "详情请访问原网页！"
-
-        if len(attachments) == 0:
-            pass
-        else:
-            list_item.projectinfo = {"attachments": attachments}
+                data_item.contenthtml = "详情请访问原网页！"
 
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == "__main__":
-    Details(redis_key="lzz:gjdtdzswpt_cgxmxx").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_gjdtdzswpt_fzxx_zzgg/非招信息-终止公告-列表页.py

@@ -6,33 +6,21 @@ Created on 2025-03-12
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
 from collections import namedtuple
+
+import feapder
 from feapder.utils.tools import get_today_of_day
-from selenium.webdriver import ActionChains
-import time
+from items.spider_item import BidingListItem
 
 
-class Feapder(feapder.BiddingListSpider):
-    __custom_setting__ = dict(
-        WEBDRIVER=dict(
-            pool_size=1,
-            headless=True,
-            driver_type="FIREFOX",
-            usages_local_driver=True
-        )
-    )
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
         self.site = "国家电投电子商务平台"
-
         self.menus = [
             Menu('非招信息-终止公告', 'a_gjdtdzswpt_fzxx_zzgg', 2),
         ]
-
         self.headers = {
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
             "Accept-Language": "zh-CN,zh;q=0.9",
@@ -42,43 +30,31 @@ class Feapder(feapder.BiddingListSpider):
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
         }
 
-        self.time = get_today_of_day()
-
+        self.date = get_today_of_day()
         self.cookies = {}
 
     def start_requests(self):
         for menu in self.menus:
-            start_url = f"https://ebid.espic.com.cn/newgdtcms//category/iframe.html?dates=300&categoryId=6&tenderMethod=00&tabName=&page=1&time={self.time}"
+            start_url = f"https://ebid.espic.com.cn/newgdtcms//category/iframe.html?dates=300&categoryId=6&tenderMethod=00&tabName=&page=1&time={self.date}"
             yield feapder.Request(url=start_url, item=menu._asdict(), page=1,
                                   render_time=10, render=True)
 
     def download_midware(self, request):
         page = request.page
-        url = f"https://ebid.espic.com.cn/newgdtcms//category/iframe.html?dates=300&categoryId=6&tenderMethod=00&tabName=&page={page}&time={self.time}"
+        url = f"https://ebid.espic.com.cn/newgdtcms//category/iframe.html?dates=300&categoryId=6&tenderMethod=00&tabName=&page={page}&time={self.date}"
         request.url = url
         request.headers = self.headers
         request.cookies = self.cookies
 
-
-    def parse(self, request, response):
-        time.sleep(10)
+    def validate(self, request, response):
         driver = response.browser
-        if "请拖动滑块解锁" in response.text:
-            for i in range(2):
-                try:
-                    slider = driver.find_element_by_xpath('//div[@class="btn"]')
-                    if slider.is_displayed():
-                        ActionChains(driver).click_and_hold(on_element=slider).perform()
-                        ActionChains(driver).move_by_offset(xoffset=260, yoffset=0).perform()
-                        ActionChains(driver).pause(1).release().perform()
-                except Exception as e:
-                    pass
-                time.sleep(2)
-                if '请拖动滑块解锁' in driver.page_source:
-                    continue
-                else:
-                    break
+        ele = driver.tab.ele('x://div[@id="js-challenge"]', timeout=1)
+        if ele:
+            ele.wait.deleted()
+
         self.cookies = driver.cookies
+
+    def parse(self, request, response):
         menu = request.item
         info_list = response.xpath('//ul[@class="newslist"]/li')
         for info in info_list:
@@ -89,7 +65,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "全国"      # 省份
             city = ""     # 城市
 
-            list_item = MgpListItem()  # 存储数据的管道
+            list_item = BidingListItem()  # 存储数据的管道
             list_item.href = href  # 标书链接
             list_item.unique_key = ('title', 'href')
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -113,4 +89,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gjdtdzswpt_cgxmxx", user="414").start()
+    Spider(redis_key="lzz:gjdtdzswpt_cgxmxx").start()

a_hnjgjzcglypt_zhbgg/中标公示-详情页.py

@@ -234,7 +234,7 @@ class Spider(feapder.BiddingDetailSpider):
                         file_type = files_info.get("file_type")
 
                     if request.proxies:
-                        fpx = request.proxies()
+                        fpx = request.get_proxies()
                     else:
                         fpx = False
 

a_obyg_cjgg/公告-详情页.py

@@ -6,13 +6,14 @@ Created on 2025-04-18
 ---------
 @author: lzz
 """
+import json
+import re
+
 import feapder
 from items.spider_item import DataBakItem
 from untils.tools import get_proxy
-from create_ck import cr_ck,ff
-import json
-import re
 
+from create_ck import cr_ck, ff
 
 headers = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -27,13 +28,13 @@ headers = {
     "X-Requested-With": "XMLHttpRequest",
 }
 
-class Details(feapder.BiddingDetailSpider):
+
+class Spider(feapder.BiddingDetailSpider):
 
     cookies = {}
-    ct = 0
     proxy = get_proxy()
     def start_requests(self):
-        data_list = self.get_tasks_by_mongodb(limit=200)
+        data_list = self.get_tasks_by_rabbitmq(limit=200)
         for item in data_list:
             request_params = item.get("request_params")
             if item.get('channel') == "采购公告":
@@ -51,12 +52,12 @@ class Details(feapder.BiddingDetailSpider):
         dhref = request.item.get('href')
 
         rfqMethod = "".join(re.findall('/rfqMethod=(.*?)/',dhref))
-        id = "".join(re.findall('/id=(.*?)/',dhref))
+        id_ = "".join(re.findall('/id=(.*?)/', dhref))
         data = {
             "page": 1,
             "rows": 100,
             "rfqMethod": rfqMethod,
-            "id": id,
+            "id": id_,
         }
         data = json.dumps(data, separators=(',', ':'))
 
@@ -65,198 +66,197 @@ class Details(feapder.BiddingDetailSpider):
         request.headers = headers
         request.proxies = self.proxy
 
-    def detail_get(self, request, response):
-        if self.ct > 5:
-            return
+    def validate(self, request, response):
         if response.status_code != 200:
-            self.ct += 1
-            self.cookies = {}
-            self.proxy = get_proxy()
-            yield request
-        else:
-            self.ct = 0
-            items = request.item
-            list_item = DataBakItem(**items)
+            raise ConnectionRefusedError
 
-            dt = response.json.get('data')
-            requestDto = dt.get('requestDto')
-            requestItemList = dt.get('requestItemList')
+    def detail_get(self, request, response):
+        items = request.item
+        data_item = DataBakItem(**items)
 
-            wl_html = ""
-            if requestItemList:
-                for info in requestItemList:
-                    temp = f'''
-                    <tr data-index="0">
-                        <td data-field="materialNo" data-key="1-0-0" align="center">
-                            <div>{info.get('materialNo','')}</div>
-                        </td>
-                        <td data-field="materialName" data-key="1-0-1" align="center">
-                            <div>{info.get('materialName','')}</div>
-                        </td>
-                        <td data-field="characters" data-key="1-0-2" align="center">
-                            <div>{info.get('characters','')}</div>
-                        </td>
-                        <td data-field="producer" data-key="1-0-3" align="center">
-                            <div>{info.get('producer','')}</div>
-                        </td>
-                        <td data-field="requestAmount" data-key="1-0-4" align="center">
-                            <div>{int(info.get('requestAmount',0))}</div>
-                        </td>
-                        <td data-field="unit" data-key="1-0-5" align="center">
-                            <div>{info.get('unit','')}</div>
-                        </td>
-                        <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
-                            <div>{ff(info.get('requestDeliveryDate',''))}</div>
-                        </td>
-                        <td data-field="memo" data-key="1-0-7" align="center">
-                            <div>{info.get('memo','')}</div>
-                        </td>
-                    </tr>
-                    '''
-                    wl_html += temp
+        dt = response.json.get('data')
+        requestDto = dt.get('requestDto')
+        requestItemList = dt.get('requestItemList')
 
-            if requestDto.get('publicitySourceFlag') == 'DYLY':
-                xjdxq = f'''
-                <div>五、采用单一来源采购方式的原因及说明：
-                    <div>{requestDto.get('chooseSingleModeReason','无')}</div>
-                </div>
-                <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
-                <p>七、公示期限：
-                    {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
-                    {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
-                </p>
-                 <p>八、备注：{requestDto.get('memo','')}</p>
-                '''
-            else:
-                xjdxq = f'''
-                <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
-                <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
-                <p>七、资质要求：{requestDto.get('qualifications','')} </p>
-                <p>八、备注：{requestDto.get('memo','')}</p></div>
+        wl_html = ""
+        if requestItemList:
+            for info in requestItemList:
+                temp = f'''
+                <tr data-index="0">
+                    <td data-field="materialNo" data-key="1-0-0" align="center">
+                        <div>{info.get('materialNo','')}</div>
+                    </td>
+                    <td data-field="materialName" data-key="1-0-1" align="center">
+                        <div>{info.get('materialName','')}</div>
+                    </td>
+                    <td data-field="characters" data-key="1-0-2" align="center">
+                        <div>{info.get('characters','')}</div>
+                    </td>
+                    <td data-field="producer" data-key="1-0-3" align="center">
+                        <div>{info.get('producer','')}</div>
+                    </td>
+                    <td data-field="requestAmount" data-key="1-0-4" align="center">
+                        <div>{int(info.get('requestAmount',0))}</div>
+                    </td>
+                    <td data-field="unit" data-key="1-0-5" align="center">
+                        <div>{info.get('unit','')}</div>
+                    </td>
+                    <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
+                        <div>{ff(info.get('requestDeliveryDate',''))}</div>
+                    </td>
+                    <td data-field="memo" data-key="1-0-7" align="center">
+                        <div>{info.get('memo','')}</div>
+                    </td>
+                </tr>
                 '''
+                wl_html += temp
 
-            html = f'''
-            <div class="left-body">
-                <div id="requestInfo">
-                    <div class="label-block" id="rfqMethodDiv">
-                        <span>询比价</span></div>
-                    <div id="tableRelevantDataTplDiv">
-                        <table class="timetable" id="tableRelevantData">
-                            <tbody>
-                            <tr>
-                                <td class="odd">询价单号</td>
-                                <td>{requestDto.get('ouRfqNum')}</td>
-                                <td class="odd">采购方式</td>
-                                <td>{requestDto.get('publicBiddingFlag')}</td>
-                            </tr>
-                            <tr>
-                                <td class="odd"> 报名截止时间</td>
-                                <td>{ff(requestDto.get('registrationEndDate'))}</td>
-                                <td class="odd">报价截至时间</td>
-                                <td>{ff(requestDto.get('quotationEndDate'))}</td>
-                            </tr>
-                            </tbody>
-                        </table>
-                    </div>
+        if requestDto.get('publicitySourceFlag') == 'DYLY':
+            xjdxq = f'''
+            <div>五、采用单一来源采购方式的原因及说明：
+                <div>{requestDto.get('chooseSingleModeReason','无')}</div>
+            </div>
+            <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
+            <p>七、公示期限：
+                {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
+                {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
+            </p>
+             <p>八、备注：{requestDto.get('memo','')}</p>
+            '''
+        else:
+            xjdxq = f'''
+            <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
+            <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
+            <p>七、资质要求：{requestDto.get('qualifications','')} </p>
+            <p>八、备注：{requestDto.get('memo','')}</p></div>
+            '''
+
+        html = f'''
+        <div class="left-body">
+            <div id="requestInfo">
+                <div class="label-block" id="rfqMethodDiv">
+                    <span>询比价</span></div>
+                <div id="tableRelevantDataTplDiv">
+                    <table class="timetable" id="tableRelevantData">
+                        <tbody>
+                        <tr>
+                            <td class="odd">询价单号</td>
+                            <td>{requestDto.get('ouRfqNum')}</td>
+                            <td class="odd">采购方式</td>
+                            <td>{requestDto.get('publicBiddingFlag')}</td>
+                        </tr>
+                        <tr>
+                            <td class="odd"> 报名截止时间</td>
+                            <td>{ff(requestDto.get('registrationEndDate'))}</td>
+                            <td class="odd">报价截至时间</td>
+                            <td>{ff(requestDto.get('quotationEndDate'))}</td>
+                        </tr>
+                        </tbody>
+                    </table>
                 </div>
-                <div class="left-b">
-                    <div class="head-txt">
-                        <span>物料信息</span>
-                    </div>
-                    <div>
-                        <table></table>
-                        <div style=" ">
-                            <div class="layui-table-box">
-                                <div class="layui-table-header">
-                                    <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
-                                        <thead>
-                                        <tr>
-                                            <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
-                                                <div align="center"><span>物料代码</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
-                                                <div align="center"><span>物料名称</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
-                                                <div align="center"><span>规格型号</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
-                                                <div align="center"><span>品牌</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
-                                                <div align="center"><span>采购数量</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
-                                                <div align="center"><span>计量单位</span></div>
-                                            </th>
-                                            <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
-                                               >
-                                                <div align="center"><span>要求交货期</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
-                                                <div align="center"><span>备注</span>
-                                                </div>
-                                            </th>
-                                        </tr>
-                                        </thead>
-                                        <tbody>
-                                        {wl_html}
-                                        </tbody>
-                                    </table>
-                                </div>
+            </div>
+            <div class="left-b">
+                <div class="head-txt">
+                    <span>物料信息</span>
+                </div>
+                <div>
+                    <table></table>
+                    <div style=" ">
+                        <div class="layui-table-box">
+                            <div class="layui-table-header">
+                                <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
+                                    <thead>
+                                    <tr>
+                                        <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
+                                            <div align="center"><span>物料代码</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
+                                            <div align="center"><span>物料名称</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
+                                            <div align="center"><span>规格型号</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
+                                            <div align="center"><span>品牌</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
+                                            <div align="center"><span>采购数量</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
+                                            <div align="center"><span>计量单位</span></div>
+                                        </th>
+                                        <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
+                                           >
+                                            <div align="center"><span>要求交货期</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
+                                            <div align="center"><span>备注</span>
+                                            </div>
+                                        </th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    {wl_html}
+                                    </tbody>
+                                </table>
                             </div>
                         </div>
                     </div>
                 </div>
-            
-                <div class="left-b">
-                    <div class="head-txt"><span>询单要求</span></div>
-                    <div class="inb inb-1">
-                        <div class="left-title">询价条款</div>
-                        <div class="left-content" id="termsDiv">
-                            <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
-                                <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
-                                <div>三、商务条款：
-                                    <div>{requestDto.get('requestBusiTerms','')}</div>
-                                </div>
-                                <div>四、技术条款：
-                                    <div>{requestDto.get('requestTecTerms','')}</div>
-                                </div>
-                                {xjdxq}
-                        </div>
+            </div>
+        
+            <div class="left-b">
+                <div class="head-txt"><span>询单要求</span></div>
+                <div class="inb inb-1">
+                    <div class="left-title">询价条款</div>
+                    <div class="left-content" id="termsDiv">
+                        <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
+                            <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
+                            <div>三、商务条款：
+                                <div>{requestDto.get('requestBusiTerms','')}</div>
+                            </div>
+                            <div>四、技术条款：
+                                <div>{requestDto.get('requestTecTerms','')}</div>
+                            </div>
+                            {xjdxq}
                     </div>
                 </div>
             </div>
-            '''
+        </div>
+        '''
+
+        data_item.contenthtml = html.replace('None','')
 
-            list_item.contenthtml = html.replace('None','')
+        # 附件下载需登录，暂未下载
+        # attachments = {}
+        # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
+        # if file_dict:
+        #     for info in file_dict:
+        #         file_url = info.xpath('./@href|./@data-key').extract_first("")
+        #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
+        #         if "http" not in file_url:
+        #             file_url = "https://www.obei.com.cn" + file_url
+        #         file_type = extract_file_type(file_name, file_url)
+        #         if file_type:
+        #             attachment = AttachmentDownloader().fetch_attachment(
+        #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.get_proxies())
+        #             attachments[str(len(attachments) + 1)] = attachment
+        #
+        # if attachments:
+        #     data_item.projectinfo = {"attachments": attachments}
 
-            # 附件下载需登录，暂未下载
-            # attachments = {}
-            # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
-            # if file_dict:
-            #     for info in file_dict:
-            #         file_url = info.xpath('./@href|./@data-key').extract_first("")
-            #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
-            #         if "http" not in file_url:
-            #             file_url = "https://www.obei.com.cn" + file_url
-            #         file_type = extract_file_type(file_name, file_url)
-            #         if file_type:
-            #             attachment = AttachmentDownloader().fetch_attachment(
-            #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.proxies())
-            #             attachments[str(len(attachments) + 1)] = attachment
-            #
-            # if attachments:
-            #     list_item.projectinfo = {"attachments": attachments}
+        yield data_item
 
-            yield list_item
+    def exception_request(self, request, response):
+        self.cookies = {}
+        self.proxy = get_proxy()
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:obyg_dylygs").start()
+    Spider(redis_key="lzz:obyg_dylygs").start()

a_obyg_dylygs/公告-详情页.py

@@ -6,12 +6,14 @@ Created on 2025-04-18
 ---------
 @author: lzz
 """
+import json
+import re
+
 import feapder
 from items.spider_item import DataBakItem
 from untils.tools import get_proxy
-from create_ck import cr_ck,ff
-import json
-import re
+
+from create_ck import cr_ck, ff
 
 
 headers = {
@@ -27,13 +29,13 @@ headers = {
     "X-Requested-With": "XMLHttpRequest",
 }
 
-class Details(feapder.BiddingDetailSpider):
+
+class Spider(feapder.BiddingDetailSpider):
 
     cookies = {}
-    ct = 0
     proxy = get_proxy()
     def start_requests(self):
-        data_list = self.get_tasks_by_mongodb(limit=200)
+        data_list = self.get_tasks_by_rabbitmq(limit=200)
         for item in data_list:
             request_params = item.get("request_params")
             if item.get('channel') == "采购公告":
@@ -65,198 +67,197 @@ class Details(feapder.BiddingDetailSpider):
         request.headers = headers
         request.proxies = self.proxy
 
-    def detail_get(self, request, response):
-        if self.ct > 5:
-            return
+    def validate(self, request, response):
         if response.status_code != 200:
-            self.ct += 1
-            self.cookies = {}
-            self.proxy = get_proxy()
-            yield request
-        else:
-            self.ct = 0
-            items = request.item
-            list_item = DataBakItem(**items)
+            raise ConnectionRefusedError
 
-            dt = response.json.get('data')
-            requestDto = dt.get('requestDto')
-            requestItemList = dt.get('requestItemList')
+    def detail_get(self, request, response):
+        items = request.item
+        data_item = DataBakItem(**items)
 
-            wl_html = ""
-            if requestItemList:
-                for info in requestItemList:
-                    temp = f'''
-                    <tr data-index="0">
-                        <td data-field="materialNo" data-key="1-0-0" align="center">
-                            <div>{info.get('materialNo','')}</div>
-                        </td>
-                        <td data-field="materialName" data-key="1-0-1" align="center">
-                            <div>{info.get('materialName','')}</div>
-                        </td>
-                        <td data-field="characters" data-key="1-0-2" align="center">
-                            <div>{info.get('characters','')}</div>
-                        </td>
-                        <td data-field="producer" data-key="1-0-3" align="center">
-                            <div>{info.get('producer','')}</div>
-                        </td>
-                        <td data-field="requestAmount" data-key="1-0-4" align="center">
-                            <div>{int(info.get('requestAmount',0))}</div>
-                        </td>
-                        <td data-field="unit" data-key="1-0-5" align="center">
-                            <div>{info.get('unit','')}</div>
-                        </td>
-                        <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
-                            <div>{ff(info.get('requestDeliveryDate',''))}</div>
-                        </td>
-                        <td data-field="memo" data-key="1-0-7" align="center">
-                            <div>{info.get('memo','')}</div>
-                        </td>
-                    </tr>
-                    '''
-                    wl_html += temp
+        dt = response.json.get('data')
+        requestDto = dt.get('requestDto')
+        requestItemList = dt.get('requestItemList')
 
-            if requestDto.get('publicitySourceFlag') == 'DYLY':
-                xjdxq = f'''
-                <div>五、采用单一来源采购方式的原因及说明：
-                    <div>{requestDto.get('chooseSingleModeReason','无')}</div>
-                </div>
-                <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
-                <p>七、公示期限：
-                    {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
-                    {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
-                </p>
-                 <p>八、备注：{requestDto.get('memo','')}</p>
-                '''
-            else:
-                xjdxq = f'''
-                <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
-                <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
-                <p>七、资质要求：{requestDto.get('qualifications','')} </p>
-                <p>八、备注：{requestDto.get('memo','')}</p></div>
+        wl_html = ""
+        if requestItemList:
+            for info in requestItemList:
+                temp = f'''
+                <tr data-index="0">
+                    <td data-field="materialNo" data-key="1-0-0" align="center">
+                        <div>{info.get('materialNo','')}</div>
+                    </td>
+                    <td data-field="materialName" data-key="1-0-1" align="center">
+                        <div>{info.get('materialName','')}</div>
+                    </td>
+                    <td data-field="characters" data-key="1-0-2" align="center">
+                        <div>{info.get('characters','')}</div>
+                    </td>
+                    <td data-field="producer" data-key="1-0-3" align="center">
+                        <div>{info.get('producer','')}</div>
+                    </td>
+                    <td data-field="requestAmount" data-key="1-0-4" align="center">
+                        <div>{int(info.get('requestAmount',0))}</div>
+                    </td>
+                    <td data-field="unit" data-key="1-0-5" align="center">
+                        <div>{info.get('unit','')}</div>
+                    </td>
+                    <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
+                        <div>{ff(info.get('requestDeliveryDate',''))}</div>
+                    </td>
+                    <td data-field="memo" data-key="1-0-7" align="center">
+                        <div>{info.get('memo','')}</div>
+                    </td>
+                </tr>
                 '''
+                wl_html += temp
 
-            html = f'''
-            <div class="left-body">
-                <div id="requestInfo">
-                    <div class="label-block" id="rfqMethodDiv">
-                        <span>询比价</span></div>
-                    <div id="tableRelevantDataTplDiv">
-                        <table class="timetable" id="tableRelevantData">
-                            <tbody>
-                            <tr>
-                                <td class="odd">询价单号</td>
-                                <td>{requestDto.get('ouRfqNum')}</td>
-                                <td class="odd">采购方式</td>
-                                <td>{requestDto.get('publicBiddingFlag')}</td>
-                            </tr>
-                            <tr>
-                                <td class="odd"> 报名截止时间</td>
-                                <td>{ff(requestDto.get('registrationEndDate'))}</td>
-                                <td class="odd">报价截至时间</td>
-                                <td>{ff(requestDto.get('quotationEndDate'))}</td>
-                            </tr>
-                            </tbody>
-                        </table>
-                    </div>
+        if requestDto.get('publicitySourceFlag') == 'DYLY':
+            xjdxq = f'''
+            <div>五、采用单一来源采购方式的原因及说明：
+                <div>{requestDto.get('chooseSingleModeReason','无')}</div>
+            </div>
+            <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
+            <p>七、公示期限：
+                {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
+                {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
+            </p>
+             <p>八、备注：{requestDto.get('memo','')}</p>
+            '''
+        else:
+            xjdxq = f'''
+            <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
+            <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
+            <p>七、资质要求：{requestDto.get('qualifications','')} </p>
+            <p>八、备注：{requestDto.get('memo','')}</p></div>
+            '''
+
+        html = f'''
+        <div class="left-body">
+            <div id="requestInfo">
+                <div class="label-block" id="rfqMethodDiv">
+                    <span>询比价</span></div>
+                <div id="tableRelevantDataTplDiv">
+                    <table class="timetable" id="tableRelevantData">
+                        <tbody>
+                        <tr>
+                            <td class="odd">询价单号</td>
+                            <td>{requestDto.get('ouRfqNum')}</td>
+                            <td class="odd">采购方式</td>
+                            <td>{requestDto.get('publicBiddingFlag')}</td>
+                        </tr>
+                        <tr>
+                            <td class="odd"> 报名截止时间</td>
+                            <td>{ff(requestDto.get('registrationEndDate'))}</td>
+                            <td class="odd">报价截至时间</td>
+                            <td>{ff(requestDto.get('quotationEndDate'))}</td>
+                        </tr>
+                        </tbody>
+                    </table>
                 </div>
-                <div class="left-b">
-                    <div class="head-txt">
-                        <span>物料信息</span>
-                    </div>
-                    <div>
-                        <table></table>
-                        <div style=" ">
-                            <div class="layui-table-box">
-                                <div class="layui-table-header">
-                                    <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
-                                        <thead>
-                                        <tr>
-                                            <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
-                                                <div align="center"><span>物料代码</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
-                                                <div align="center"><span>物料名称</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
-                                                <div align="center"><span>规格型号</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
-                                                <div align="center"><span>品牌</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
-                                                <div align="center"><span>采购数量</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
-                                                <div align="center"><span>计量单位</span></div>
-                                            </th>
-                                            <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
-                                               >
-                                                <div align="center"><span>要求交货期</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
-                                                <div align="center"><span>备注</span>
-                                                </div>
-                                            </th>
-                                        </tr>
-                                        </thead>
-                                        <tbody>
-                                        {wl_html}
-                                        </tbody>
-                                    </table>
-                                </div>
+            </div>
+            <div class="left-b">
+                <div class="head-txt">
+                    <span>物料信息</span>
+                </div>
+                <div>
+                    <table></table>
+                    <div style=" ">
+                        <div class="layui-table-box">
+                            <div class="layui-table-header">
+                                <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
+                                    <thead>
+                                    <tr>
+                                        <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
+                                            <div align="center"><span>物料代码</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
+                                            <div align="center"><span>物料名称</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
+                                            <div align="center"><span>规格型号</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
+                                            <div align="center"><span>品牌</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
+                                            <div align="center"><span>采购数量</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
+                                            <div align="center"><span>计量单位</span></div>
+                                        </th>
+                                        <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
+                                           >
+                                            <div align="center"><span>要求交货期</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
+                                            <div align="center"><span>备注</span>
+                                            </div>
+                                        </th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    {wl_html}
+                                    </tbody>
+                                </table>
                             </div>
                         </div>
                     </div>
                 </div>
-            
-                <div class="left-b">
-                    <div class="head-txt"><span>询单要求</span></div>
-                    <div class="inb inb-1">
-                        <div class="left-title">询价条款</div>
-                        <div class="left-content" id="termsDiv">
-                            <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
-                                <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
-                                <div>三、商务条款：
-                                    <div>{requestDto.get('requestBusiTerms','')}</div>
-                                </div>
-                                <div>四、技术条款：
-                                    <div>{requestDto.get('requestTecTerms','')}</div>
-                                </div>
-                                {xjdxq}
-                        </div>
+            </div>
+        
+            <div class="left-b">
+                <div class="head-txt"><span>询单要求</span></div>
+                <div class="inb inb-1">
+                    <div class="left-title">询价条款</div>
+                    <div class="left-content" id="termsDiv">
+                        <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
+                            <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
+                            <div>三、商务条款：
+                                <div>{requestDto.get('requestBusiTerms','')}</div>
+                            </div>
+                            <div>四、技术条款：
+                                <div>{requestDto.get('requestTecTerms','')}</div>
+                            </div>
+                            {xjdxq}
                     </div>
                 </div>
             </div>
-            '''
+        </div>
+        '''
+
+        data_item.contenthtml = html.replace('None','')
 
-            list_item.contenthtml = html.replace('None','')
+        # 附件下载需登录，暂未下载
+        # attachments = {}
+        # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
+        # if file_dict:
+        #     for info in file_dict:
+        #         file_url = info.xpath('./@href|./@data-key').extract_first("")
+        #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
+        #         if "http" not in file_url:
+        #             file_url = "https://www.obei.com.cn" + file_url
+        #         file_type = extract_file_type(file_name, file_url)
+        #         if file_type:
+        #             attachment = AttachmentDownloader().fetch_attachment(
+        #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.get_proxies())
+        #             attachments[str(len(attachments) + 1)] = attachment
+        #
+        # if attachments:
+        #     data_item.projectinfo = {"attachments": attachments}
 
-            # 附件下载需登录，暂未下载
-            # attachments = {}
-            # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
-            # if file_dict:
-            #     for info in file_dict:
-            #         file_url = info.xpath('./@href|./@data-key').extract_first("")
-            #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
-            #         if "http" not in file_url:
-            #             file_url = "https://www.obei.com.cn" + file_url
-            #         file_type = extract_file_type(file_name, file_url)
-            #         if file_type:
-            #             attachment = AttachmentDownloader().fetch_attachment(
-            #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.proxies())
-            #             attachments[str(len(attachments) + 1)] = attachment
-            #
-            # if attachments:
-            #     list_item.projectinfo = {"attachments": attachments}
+        yield data_item
 
-            yield list_item
+    def exception_request(self, request, response):
+        self.cookies = {}
+        self.proxy = get_proxy()
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:obyg_dylygs").start()
+    Spider(redis_key="lzz:obyg_dylygs").start()

a_oycg_gkcggg/公告-详情页.py

@@ -6,13 +6,14 @@ Created on 2025-04-18
 ---------
 @author: lzz
 """
+import json
+import re
+
 import feapder
 from items.spider_item import DataBakItem
 from untils.tools import get_proxy
-from create_ck import cr_ck,ff
-import json
-import re
 
+from create_ck import cr_ck, ff
 
 headers = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -27,13 +28,13 @@ headers = {
     "X-Requested-With": "XMLHttpRequest",
 }
 
-class Details(feapder.BiddingDetailSpider):
+
+class Spider(feapder.BiddingDetailSpider):
 
     cookies = {}
-    ct = 0
     proxy = get_proxy()
     def start_requests(self):
-        data_list = self.get_tasks_by_mongodb(limit=200)
+        data_list = self.get_tasks_by_rabbitmq(limit=200)
         for item in data_list:
             request_params = item.get("request_params")
             if item.get('channel') == "采购公告":
@@ -51,12 +52,12 @@ class Details(feapder.BiddingDetailSpider):
         dhref = request.item.get('href')
 
         rfqMethod = "".join(re.findall('/rfqMethod=(.*?)/',dhref))
-        id = "".join(re.findall('/id=(.*?)/',dhref))
+        id_ = "".join(re.findall('/id=(.*?)/', dhref))
         data = {
             "page": 1,
             "rows": 100,
             "rfqMethod": rfqMethod,
-            "id": id,
+            "id": id_,
         }
         data = json.dumps(data, separators=(',', ':'))
 
@@ -65,198 +66,197 @@ class Details(feapder.BiddingDetailSpider):
         request.headers = headers
         request.proxies = self.proxy
 
-    def detail_get(self, request, response):
-        if self.ct > 5:
-            return
+    def validate(self, request, response):
         if response.status_code != 200:
-            self.ct += 1
-            self.cookies = {}
-            self.proxy = get_proxy()
-            yield request
-        else:
-            self.ct = 0
-            items = request.item
-            list_item = DataBakItem(**items)
+            raise ConnectionRefusedError
 
-            dt = response.json.get('data')
-            requestDto = dt.get('requestDto')
-            requestItemList = dt.get('requestItemList')
+    def detail_get(self, request, response):
+        items = request.item
+        data_item = DataBakItem(**items)
 
-            wl_html = ""
-            if requestItemList:
-                for info in requestItemList:
-                    temp = f'''
-                    <tr data-index="0">
-                        <td data-field="materialNo" data-key="1-0-0" align="center">
-                            <div>{info.get('materialNo','')}</div>
-                        </td>
-                        <td data-field="materialName" data-key="1-0-1" align="center">
-                            <div>{info.get('materialName','')}</div>
-                        </td>
-                        <td data-field="characters" data-key="1-0-2" align="center">
-                            <div>{info.get('characters','')}</div>
-                        </td>
-                        <td data-field="producer" data-key="1-0-3" align="center">
-                            <div>{info.get('producer','')}</div>
-                        </td>
-                        <td data-field="requestAmount" data-key="1-0-4" align="center">
-                            <div>{int(info.get('requestAmount',0))}</div>
-                        </td>
-                        <td data-field="unit" data-key="1-0-5" align="center">
-                            <div>{info.get('unit','')}</div>
-                        </td>
-                        <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
-                            <div>{ff(info.get('requestDeliveryDate',''))}</div>
-                        </td>
-                        <td data-field="memo" data-key="1-0-7" align="center">
-                            <div>{info.get('memo','')}</div>
-                        </td>
-                    </tr>
-                    '''
-                    wl_html += temp
+        dt = response.json.get('data')
+        requestDto = dt.get('requestDto')
+        requestItemList = dt.get('requestItemList')
 
-            if requestDto.get('publicitySourceFlag') == 'DYLY':
-                xjdxq = f'''
-                <div>五、采用单一来源采购方式的原因及说明：
-                    <div>{requestDto.get('chooseSingleModeReason','无')}</div>
-                </div>
-                <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
-                <p>七、公示期限：
-                    {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
-                    {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
-                </p>
-                 <p>八、备注：{requestDto.get('memo','')}</p>
-                '''
-            else:
-                xjdxq = f'''
-                <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
-                <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
-                <p>七、资质要求：{requestDto.get('qualifications','')} </p>
-                <p>八、备注：{requestDto.get('memo','')}</p></div>
+        wl_html = ""
+        if requestItemList:
+            for info in requestItemList:
+                temp = f'''
+                <tr data-index="0">
+                    <td data-field="materialNo" data-key="1-0-0" align="center">
+                        <div>{info.get('materialNo','')}</div>
+                    </td>
+                    <td data-field="materialName" data-key="1-0-1" align="center">
+                        <div>{info.get('materialName','')}</div>
+                    </td>
+                    <td data-field="characters" data-key="1-0-2" align="center">
+                        <div>{info.get('characters','')}</div>
+                    </td>
+                    <td data-field="producer" data-key="1-0-3" align="center">
+                        <div>{info.get('producer','')}</div>
+                    </td>
+                    <td data-field="requestAmount" data-key="1-0-4" align="center">
+                        <div>{int(info.get('requestAmount',0))}</div>
+                    </td>
+                    <td data-field="unit" data-key="1-0-5" align="center">
+                        <div>{info.get('unit','')}</div>
+                    </td>
+                    <td data-field="requestDeliveryDate" data-key="1-0-6" align="center">
+                        <div>{ff(info.get('requestDeliveryDate',''))}</div>
+                    </td>
+                    <td data-field="memo" data-key="1-0-7" align="center">
+                        <div>{info.get('memo','')}</div>
+                    </td>
+                </tr>
                 '''
+                wl_html += temp
 
-            html = f'''
-            <div class="left-body">
-                <div id="requestInfo">
-                    <div class="label-block" id="rfqMethodDiv">
-                        <span>询比价</span></div>
-                    <div id="tableRelevantDataTplDiv">
-                        <table class="timetable" id="tableRelevantData">
-                            <tbody>
-                            <tr>
-                                <td class="odd">询价单号</td>
-                                <td>{requestDto.get('ouRfqNum')}</td>
-                                <td class="odd">采购方式</td>
-                                <td>{requestDto.get('publicBiddingFlag')}</td>
-                            </tr>
-                            <tr>
-                                <td class="odd"> 报名截止时间</td>
-                                <td>{ff(requestDto.get('registrationEndDate'))}</td>
-                                <td class="odd">报价截至时间</td>
-                                <td>{ff(requestDto.get('quotationEndDate'))}</td>
-                            </tr>
-                            </tbody>
-                        </table>
-                    </div>
+        if requestDto.get('publicitySourceFlag') == 'DYLY':
+            xjdxq = f'''
+            <div>五、采用单一来源采购方式的原因及说明：
+                <div>{requestDto.get('chooseSingleModeReason','无')}</div>
+            </div>
+            <p>六、拟定供应商：{requestDto.get('draftSupplierName','无')}</p>
+            <p>七、公示期限：
+                {{d.publicityTermMap.issueDate}} 至 {{d.publicityTermMap.publicityEndDate}}
+                {ff(requestDto.get('publicityTermMap', {}).get('publicityTermMap'))} 至{ff(requestDto.get('publicityTermMap', {}).get('publicityEndDate'))}
+            </p>
+             <p>八、备注：{requestDto.get('memo','')}</p>
+            '''
+        else:
+            xjdxq = f'''
+            <p>五、注册资本必须大于等于{int(requestDto.get('regcapital',0))}万元</p>
+            <p>六、报名要求：{requestDto.get('requirementDesc','')} </p>
+            <p>七、资质要求：{requestDto.get('qualifications','')} </p>
+            <p>八、备注：{requestDto.get('memo','')}</p></div>
+            '''
+
+        html = f'''
+        <div class="left-body">
+            <div id="requestInfo">
+                <div class="label-block" id="rfqMethodDiv">
+                    <span>询比价</span></div>
+                <div id="tableRelevantDataTplDiv">
+                    <table class="timetable" id="tableRelevantData">
+                        <tbody>
+                        <tr>
+                            <td class="odd">询价单号</td>
+                            <td>{requestDto.get('ouRfqNum')}</td>
+                            <td class="odd">采购方式</td>
+                            <td>{requestDto.get('publicBiddingFlag')}</td>
+                        </tr>
+                        <tr>
+                            <td class="odd"> 报名截止时间</td>
+                            <td>{ff(requestDto.get('registrationEndDate'))}</td>
+                            <td class="odd">报价截至时间</td>
+                            <td>{ff(requestDto.get('quotationEndDate'))}</td>
+                        </tr>
+                        </tbody>
+                    </table>
                 </div>
-                <div class="left-b">
-                    <div class="head-txt">
-                        <span>物料信息</span>
-                    </div>
-                    <div>
-                        <table></table>
-                        <div style=" ">
-                            <div class="layui-table-box">
-                                <div class="layui-table-header">
-                                    <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
-                                        <thead>
-                                        <tr>
-                                            <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
-                                                <div align="center"><span>物料代码</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
-                                                <div align="center"><span>物料名称</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
-                                                <div align="center"><span>规格型号</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
-                                                <div align="center"><span>品牌</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
-                                                <div align="center"><span>采购数量</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
-                                                <div align="center"><span>计量单位</span></div>
-                                            </th>
-                                            <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
-                                               >
-                                                <div align="center"><span>要求交货期</span>
-                                                </div>
-                                            </th>
-                                            <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
-                                                <div align="center"><span>备注</span>
-                                                </div>
-                                            </th>
-                                        </tr>
-                                        </thead>
-                                        <tbody>
-                                        {wl_html}
-                                        </tbody>
-                                    </table>
-                                </div>
+            </div>
+            <div class="left-b">
+                <div class="head-txt">
+                    <span>物料信息</span>
+                </div>
+                <div>
+                    <table></table>
+                    <div style=" ">
+                        <div class="layui-table-box">
+                            <div class="layui-table-header">
+                                <table cellspacing="0" cellpadding="0" border="0" class="layui-table">
+                                    <thead>
+                                    <tr>
+                                        <th style="" data-field="materialNo" data-key="1-0-0" data-unresize="true">
+                                            <div align="center"><span>物料代码</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="materialName" data-key="1-0-1" data-unresize="true">
+                                            <div align="center"><span>物料名称</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="characters" data-key="1-0-2" data-unresize="true">
+                                            <div align="center"><span>规格型号</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="producer" data-key="1-0-3" data-unresize="true">
+                                            <div align="center"><span>品牌</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="requestAmount" data-key="1-0-4" data-unresize="true">
+                                            <div align="center"><span>采购数量</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="unit" data-key="1-0-5" data-unresize="true">
+                                            <div align="center"><span>计量单位</span></div>
+                                        </th>
+                                        <th style="" data-field="requestDeliveryDate" data-key="1-0-6" data-unresize="true"
+                                           >
+                                            <div align="center"><span>要求交货期</span>
+                                            </div>
+                                        </th>
+                                        <th style="" data-field="memo" data-key="1-0-7" data-unresize="true">
+                                            <div align="center"><span>备注</span>
+                                            </div>
+                                        </th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    {wl_html}
+                                    </tbody>
+                                </table>
                             </div>
                         </div>
                     </div>
                 </div>
-            
-                <div class="left-b">
-                    <div class="head-txt"><span>询单要求</span></div>
-                    <div class="inb inb-1">
-                        <div class="left-title">询价条款</div>
-                        <div class="left-content" id="termsDiv">
-                            <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
-                                <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
-                                <div>三、商务条款：
-                                    <div>{requestDto.get('requestBusiTerms','')}</div>
-                                </div>
-                                <div>四、技术条款：
-                                    <div>{requestDto.get('requestTecTerms','')}</div>
-                                </div>
-                                {xjdxq}
-                        </div>
+            </div>
+        
+            <div class="left-b">
+                <div class="head-txt"><span>询单要求</span></div>
+                <div class="inb inb-1">
+                    <div class="left-title">询价条款</div>
+                    <div class="left-content" id="termsDiv">
+                        <div class="terms"><p>一、交货地址：{requestDto.get('deliveryAddress')}</p>
+                            <p>二、保证金额度：{int(requestDto.get('assureMoney',0))}元</p>
+                            <div>三、商务条款：
+                                <div>{requestDto.get('requestBusiTerms','')}</div>
+                            </div>
+                            <div>四、技术条款：
+                                <div>{requestDto.get('requestTecTerms','')}</div>
+                            </div>
+                            {xjdxq}
                     </div>
                 </div>
             </div>
-            '''
+        </div>
+        '''
+
+        data_item.contenthtml = html.replace('None','')
 
-            list_item.contenthtml = html.replace('None','')
+        # 附件下载需登录，暂未下载
+        # attachments = {}
+        # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
+        # if file_dict:
+        #     for info in file_dict:
+        #         file_url = info.xpath('./@href|./@data-key').extract_first("")
+        #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
+        #         if "http" not in file_url:
+        #             file_url = "https://www.obei.com.cn" + file_url
+        #         file_type = extract_file_type(file_name, file_url)
+        #         if file_type:
+        #             attachment = AttachmentDownloader().fetch_attachment(
+        #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.get_proxies())
+        #             attachments[str(len(attachments) + 1)] = attachment
+        #
+        # if attachments:
+        #     data_item.projectinfo = {"attachments": attachments}
 
-            # 附件下载需登录，暂未下载
-            # attachments = {}
-            # file_dict = response.xpath('//div[@class="left-body"]//a[@href]')
-            # if file_dict:
-            #     for info in file_dict:
-            #         file_url = info.xpath('./@href|./@data-key').extract_first("")
-            #         file_name = info.xpath('./@href/../../../td[1]/text()').extract_first("").strip()
-            #         if "http" not in file_url:
-            #             file_url = "https://www.obei.com.cn" + file_url
-            #         file_type = extract_file_type(file_name, file_url)
-            #         if file_type:
-            #             attachment = AttachmentDownloader().fetch_attachment(
-            #                 file_name=file_name, file_type=file_type, download_url=file_url,proxies=request.proxies())
-            #             attachments[str(len(attachments) + 1)] = attachment
-            #
-            # if attachments:
-            #     list_item.projectinfo = {"attachments": attachments}
+        yield data_item
 
-            yield list_item
+    def exception_request(self, request, response):
+        self.cookies = {}
+        self.proxy = get_proxy()
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:obyg_dylygs").start()
+    Spider(redis_key="lzz:obyg_dylygs").start()

a_yctgxcglm_bggg/变更公告-列表页.py

@@ -89,7 +89,7 @@ class Spider(feapder.BiddingListSpider):
                     file_name=title,
                     file_type='pdf',
                     download_url=file_url,
-                    proxies=request.proxies()
+                    proxies=request.get_proxies()
                 )
                 if attachment.get('size'):
                     data_item.contenthtml = '详情请访问原网页！'

a_yctgxcglm_cgxq/采购需求-列表页.py

@@ -89,7 +89,7 @@ class Spider(feapder.BiddingListSpider):
                     file_name=title,
                     file_type='pdf',
                     download_url=file_url,
-                    proxies=request.proxies()
+                    proxies=request.get_proxies()
                 )
                 if attachment.get('size'):
                     data_item.contenthtml = '详情请访问原网页！'

a_yctgxcglm_cgyx/采购意向-列表页.py

@@ -89,7 +89,7 @@ class Spider(feapder.BiddingListSpider):
                     file_name=title,
                     file_type='pdf',
                     download_url=file_url,
-                    proxies=request.proxies()
+                    proxies=request.get_proxies()
                 )
                 if attachment.get('size'):
                     data_item.contenthtml = '详情请访问原网页！'

a_yctgxcglm_jggg/Yctgxcglm.py

@@ -104,7 +104,7 @@ class Spider(feapder.BiddingListSpider):
                     file_name=title,
                     file_type='pdf',
                     download_url=file_url,
-                    proxies=request.proxies()
+                    proxies=request.get_proxies()
                 )
                 if attachment.get('size'):
                     data_item.contenthtml = '详情请访问原网页！'

a_yctgxcglm_jggg/yct_details_firefox.py

@@ -7,7 +7,6 @@ Created on 2023-7-22
 @author: lzz
 """
 
-import time
 from urllib.parse import urljoin
 
 import feapder
@@ -35,12 +34,10 @@ class Spider(feapder.BiddingDetailSpider):
                                   **request_params)
 
     def detail_get(self, request, response):
-        driver = response.browser
         iframe = response.xpath('//iframe[@id="contentFrame"]').extract_first()
         if iframe:
-            driver.switch_to.frame(0)
-            time.sleep(1)
-            response = Selector(driver.page_source)
+            content_frame = response.browser.tab('#contentFrame')
+            response = Selector(content_frame.html)
 
         items = request.item
         data_item = DataBakItem(**items)
@@ -93,7 +90,7 @@ class Spider(feapder.BiddingDetailSpider):
                             file_name=file_name,
                             file_type=file_type,
                             download_url=file_url,
-                            proxies=request.proxies()
+                            proxies=request.get_proxies()
                         )
                         attachments[str(len(attachments) + 1)] = attachment
 

a_yjydzzbtbjypt_zbxx_zfcg/招标信息-政企采购.py

@@ -1,27 +1,26 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-04-27
+Created on 2025-05-30
 ---------
 @summary: 易交易电子招标投标交易平台
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from feapder.utils.tools import get_month
+import json
 import time
+from collections import namedtuple
+
 import execjs
-import json
+import feapder
+from feapder.utils.tools import get_month
+from items.spider_item import BidingListItem
 
 
 def dec_str(data):
     ex_js = '''
     CryptoJS = require('crypto-js')
     function de_str(data) {
-
         var iv = CryptoJS.enc.Latin1.parse('X9avr@CtRmcNgVkrVC9eKqemBZqzuPBf')
-    
         n = CryptoJS.AES.decrypt(data, iv, {
                 mode: CryptoJS.mode.ECB,
                 padding: CryptoJS.pad.Pkcs7
@@ -32,41 +31,33 @@ def dec_str(data):
     }
     '''
     ctx = execjs.compile(ex_js)
-    return ctx.call('de_str',data)
-
-
+    return ctx.call('de_str', data)
 
-class Feapder(feapder.BiddingListSpider):
 
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
-
-        self.site = "易交易电子招标投标交易平台"
-
         self.menus = [
-            Menu('招标信息-企业采购', 'a_yjydzzbtbjypt_zbxx_zfcg', '', 1),
+            Menu('招标信息-企业采购', 'a_yjydzzbtbjypt_zbxx_zfcg', '', 2),
         ]
-
+        self.site = "易交易电子招标投标交易平台"
         self.headers = {
             "Accept": "text/plain, */*; q=0.01",
             "Referer": "https://www.enjoy5191.com/views/public/trade.html",
             "X-Requested-With": "XMLHttpRequest",
             "sec-ch-ua-mobile": "?0",
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-            "sec-ch-ua-platform": "\"Windows\""
         }
 
-
     def start_requests(self):
+        url = "https://www.enjoy5191.com/api/SearchHandler.ashx"
         for menu in self.menus:
-            start_url = "https://www.enjoy5191.com/api/SearchHandler.ashx"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+            yield feapder.Request(url, item=menu._asdict(), page=1, proxies=False)
 
     def download_midware(self, request):
         page = request.page
         menu = request.item
-
         params = {
             "ieTime": f"{int(time.time()*1000)}",
             "BIG_TYPE": menu.get('tid'),
@@ -88,7 +79,6 @@ class Feapder(feapder.BiddingListSpider):
         request.headers = self.headers
 
     def parse(self, request, response):
-
         menu = request.item
         info_list = json.loads(dec_str(response.text)).get('data')
         for info in info_list:
@@ -96,12 +86,11 @@ class Feapder(feapder.BiddingListSpider):
             title = info.get('NAME').strip()
             create_time = info.get('PUBLISHED_TIME').strip()
 
-            cty = info.get('SEN','全国').strip()
-
-            area = cty.replace('省','')
+            cty = info.get('SEN', '全国').strip()
+            area = cty.replace('省', '')
             city = ""
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -111,12 +100,8 @@ class Feapder(feapder.BiddingListSpider):
             list_item.publishtime = create_time
             list_item.area = area                # 城市默认:全国
             list_item.city = city                # 城市 默认为空
-
-            list_item.parse = "self.detail_get"  # 详情页回调方法
-            list_item.deal_detail = ['//div[@class="tenderView"]']  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = href
-
             yield list_item
 
         # 无限翻页
@@ -125,5 +110,5 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:yjydzzbtbjypt_cqjy_zfcg").start()
+    Spider(redis_key="lzz:yjydzzbtbjypt_cqjy_zfcg").start()
 

a_yjydzzbtbjypt_zbxx_zfcg/易交易电子招标投标交易平台-详情页.py

@@ -1,20 +1,19 @@
 # -*- coding: utf-8 -*-
 """
-Created on 2025-04-27
+Created on 2025-05-30
 ---------
 @summary: 易交易电子招标投标交易平台
 ---------
 @author: lzz
 """
+import re
+import time
+
 import feapder
-from items.spider_item import DataBakItem
 from feapder.network.selector import Selector
+from items.spider_item import DataBakItem
 from untils.attachment import AttachmentDownloader
-from untils.tools import extract_file_type,text_search
-import time
-import re
-
-
+from untils.tools import extract_file_type, text_search
 
 headers = {
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -25,74 +24,102 @@ headers = {
     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
 }
 
-class Details(feapder.BiddingDetailSpider):
 
+class NoneIframeError(Exception):
+    pass
 
-    def start_requests(self):
 
+class Spider(feapder.BiddingDetailSpider):
+
+    def start_callback(self):
+        self._downloader = AttachmentDownloader()
+
+    def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             request_params = item.get("request_params")
-            yield feapder.Request(url=item.get("parse_url"), item=item,render_time=5,render=True,proxies=False,
-                                  deal_detail=item.get("deal_detail"), **request_params, callback='parse')
+            yield feapder.Request(url=item.get("parse_url"),
+                                  render=True,
+                                  proxies=False,
+                                  render_time=10,
+                                  item=item,
+                                  **request_params)
 
     def download_midware(self, request):
         request.headers = headers
 
     def parse(self, request, response):
-        driver = response.browser
-
-        gg_list = driver.find_elements_by_xpath('//div[@class="spanbox spanspeical"]/div')
-        name_list = response.xpath('//div[@class="spanbox spanspeical"]')
-        for clk,nm in zip(gg_list,name_list):
-            items = request.item
-            list_item = DataBakItem(**items)
-            old_title = list_item.title
-            old_href = list_item.href
+        items = request.item
+        tab = response.browser.tab
+        links = tab.eles('x://div[@class="spanbox spanspeical"]/div')
+        for link in links:
+            data_item = DataBakItem(**items)
+
+            data_type = next(iter(link.s_eles('x://text()')), '').strip()
+            list_title = data_item.title
+            if not re.findall(f'.*{data_type}$', list_title):
+                data_item.title = list_title + data_type
+
+            href = data_item.href
+            data_item.href = href + f"&{int(time.time())}"
+
+            # 点击元素
+            tab.ele(link).click()
+            # 等待dom加载完成
+            tab.wait.doc_loaded()
+
             attachments = {}
-            clk.click()
-            name = (nm.xpath('.//text()').extract()[0] or "").strip()
-            if not re.findall(f'.*{name}$',old_title):
-                list_item.title = old_title + name.strip()
-            list_item.href = old_href + f"&{int(time.time())}"
-            time.sleep(2)
-            rt = Selector(driver.page_source)
-            file_url = rt.xpath('//iframe/@src').extract_first("")
+            s1 = Selector(tab.html)
+            file_url = s1.xpath('//iframe/@src').extract_first("")
             file_type = extract_file_type(file_url=file_url)
             if file_url and file_type:
                 if "http" not in file_url:
                     file_url = "https://fjjsztb.enjoy5191.com" + file_url
-                attachment = AttachmentDownloader().fetch_attachment(
-                    file_name=list_item.title, file_type=file_type, download_url=file_url)
+
+                attachment = self._downloader.fetch_attachment(
+                    file_name=data_item.title,
+                    file_type=file_type,
+                    download_url=file_url
+                )
                 attachments[str(len(attachments) + 1)] = attachment
-            driver.switch_to.frame(0)
-            time.sleep(2)
 
-            xpath_list = ['//div[contains(@class,"tenderView")]','//div[contains(@class,"text-center")]',
-                          '//div[@class="app"]','//div[@id="iframe_main"]']
-            root = Selector(driver.page_source)
+            # 获取 iframe 内嵌源码
+            iframe = None
+            locs = ['#combineIframe', '.myFrame']
+            for loc in locs:
+                iframe = tab.get_frame(loc)
+                if iframe:
+                    break
+
+            if not iframe:
+                raise NoneIframeError
+
+            s2 = Selector(iframe.html)
 
             html = ""
+            xpath_list = [
+                '//div[contains(@class,"tenderView")]',
+                '//div[contains(@class,"text-center")]',
+                '//div[@class="app"]',
+                '//div[@id="iframe_main"]'
+            ]
             for xpath in xpath_list:
-                html = root.xpath(xpath).extract_first()
+                html = s2.xpath(xpath).extract_first()
                 if html:
                     break
 
-            ex_html = root.xpath('//div[@class="colorRed"]').extract_first()
+            ex_html = s2.xpath('//div[@class="colorRed"]').extract_first()
             if ex_html:
-                html = html.replace(ex_html,'')
+                html = html.replace(ex_html, '')
 
-            if attachments:
-                list_item.projectinfo = {"attachments": attachments}
+            if len(attachments) > 0:
+                data_item.projectinfo = {"attachments": attachments}
                 if text_search(html).total < 10:
                     html = "详情请访问原网页！"
 
-            list_item.contenthtml = html
-
-            driver.switch_to.parent_frame()
-
-            yield list_item
+            data_item.contenthtml = html
+            yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:yjydzzbtbjypt_cqjy_zfcg").start()
+    Spider(redis_key="lzz:yjydzzbtbjypt_cqjy_zfcg").start()

a_zgsyzbtbw_gkzbgg/中国石油招标投标网-详情页.py

@@ -90,7 +90,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             if attachments:

a_zgsyzbtbw_gkzbzbhxrgs/中国石油招标投标网-详情页.py

@@ -90,7 +90,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             if attachments:

a_zgsyzbtbw_gkzbzbjggg/中国石油招标投标网-详情页.py

@@ -90,7 +90,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             if attachments:

a_zgsyzbtbw_zgysgg/中国石油招标投标网-详情页.py

@@ -90,7 +90,7 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             if attachments:

a_zgsyzbtbw_zgysgg/资格预审公告-列表页.py

@@ -10,13 +10,13 @@ import time
 from collections import namedtuple
 
 import feapder
-from items.spider_item import MgpListItem
+from items.spider_item import BidingListItem
 from untils.tools import get_proxy
 
 from utils import ctx, get_logo, parse_item_list
 
 
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
@@ -37,7 +37,7 @@ class Feapder(feapder.BiddingListSpider):
         t_date = str(round(time.time() * 1000))
         logo_text = get_logo(t_date, proxies=proxies)
         logo = ctx().call('get_logo', logo_text)
-
+        request.logo = logo
 
         request.headers = {
             "Accept": "application/json, text/plain, */*",
@@ -63,7 +63,6 @@ class Feapder(feapder.BiddingListSpider):
         }
         request.json = ctx().call('encrypt', logo['logo1'], data)
 
-
     def parse(self, request, response):
         menu = request.item
         info_list = parse_item_list(request, response)
@@ -76,7 +75,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "全国"    # 省份
             city = ""     # 城市
 
-            list_item = MgpListItem()  # 存储数据的管道
+            list_item = BidingListItem()  # 存储数据的管道
             list_item.href = href  # 标书链接
             list_item.unique_key = ('title', 'href')
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -89,8 +88,8 @@ class Feapder(feapder.BiddingListSpider):
 
             list_item.unique_key = ('href',)
             list_item.parse = "self.detail_get"  # 详情页回调方法
-            pm = {"columnId": 2, "articleId": hid}
-            list_item.request_params = {"datapm": pm, "method":"POST"}
+            pm = {"columnId": "1", "articleId": hid}
+            list_item.request_params = {"datapm": pm, "method": "POST"}
             list_item.deal_detail = []
             list_item.proxies = False
             list_item.parse_url = "https://www.cnpcbidding.com/cms/article/details"
@@ -103,4 +102,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:zgsyzbtbw_details").start()
+    Spider(redis_key="lzz:zgsyzbtbw_details").start()

gd_gdsggzyjypt_gcjs_jggg/工程建设-中标结果-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
         self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('工程建设-中标结果', 'gd_gdsggzyjypt_gcjs_jtysgc', 5),
         ]
 
-
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -88,7 +82,7 @@ class Feapder(feapder.BiddingListSpider):
                 "bizCode": f"{tradingProcess}",
                 "projectCode": f"{projectCode}"
             }
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -99,7 +93,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -125,8 +119,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -135,5 +127,5 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gcjs", user="1122").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()
 

gd_gdsggzyjypt_gcjs_jtysgc/工程建设-中标结果-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import MgpListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
         self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('工程建设-中标结果', 'gd_gdsggzyjypt_gcjs_jtysgc', 5),
         ]
 
-
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -88,7 +82,7 @@ class Feapder(feapder.BiddingListSpider):
                 "bizCode": f"{tradingProcess}",
                 "projectCode": f"{projectCode}"
             }
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -125,8 +119,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -135,5 +127,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gcjs", user="1122").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gcjs_qtgc/工程建设-招标公告与资格预审公告-列表页.py

@@ -6,31 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import MgpListItem
 
+from gd_utils import *
 
 
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('工程建设-招标公告与资格预审公告', 'gd_gdsggzyjypt_gcjs_qtgc', 10),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -67,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -87,7 +82,7 @@ class Feapder(feapder.BiddingListSpider):
                 "bizCode": f"{tradingProcess}",
                 "projectCode": f"{projectCode}"
             }
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -124,7 +119,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
             yield list_item
 
         # 无限翻页
@@ -133,4 +127,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gcjs", user="1128").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gcjs_qtgc/工程建设-详情页.py

@@ -17,15 +17,20 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
+
+    def start_callback(self):
+        self._downloader = AttachmentDownloader()
 
     def start_requests(self):
         data_list = self.get_tasks_by_rabbitmq(limit=30)
         for item in data_list:
             request_params = item.get("request_params")
-            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,
-                                  deal_detail=item.get("deal_detail"), **request_params,
-                                  callback='parse')
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,7 +53,7 @@ class Details(feapder.BiddingDetailSpider):
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -179,12 +184,13 @@ class Details(feapder.BiddingDetailSpider):
             f_list = detail_info[-1].get('noticeFileBOList')
         except:
             f_list = None
+
         if f_list:
             ff_html = ""
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/', list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/', data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -201,9 +207,12 @@ class Details(feapder.BiddingDetailSpider):
                 ff_html += temp
                 f_type = extract_file_type(f_name, f_url)
                 if f_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=f_name,
+                        file_type=f_type,
+                        download_url=f_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -217,7 +226,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + htxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + htxx_html + ggnr_html + file_html
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
         if file_list:
@@ -226,14 +235,18 @@ class Details(feapder.BiddingDetailSpider):
                 file_url = info.xpath('./@href').extract_first()
                 file_type = extract_file_type(file_name, file_url)
                 if file_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=file_name,
+                        file_type=file_type,
+                        download_url=file_url
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
-        if attachments:
-            list_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
+
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_gcjs").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gcjs_slgc/工程建设-中标候选人公示-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import MgpListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('工程建设-中标候选人公示', 'gd_gdsggzyjypt_gcjs_slgc', 10),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -90,7 +84,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}",
                 "classify": f"{projectType}"
             }
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -127,7 +121,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
             yield list_item
 
         # 无限翻页
@@ -136,5 +129,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gcjs", user="1124").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gcjs_slgc/工程建设-详情页.py

@@ -17,15 +17,20 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
+
+    def start_callback(self):
+        self._downloader = AttachmentDownloader()
 
     def start_requests(self):
-        data_list = self.get_tasks_by_rabbitmq(limit=30)
+        data_list = self.get_tasks(limit=30)
         for item in data_list:
             request_params = item.get("request_params")
-            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,
-                                  deal_detail=item.get("deal_detail"), **request_params,
-                                  callback='parse')
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,7 +53,7 @@ class Details(feapder.BiddingDetailSpider):
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -179,12 +184,13 @@ class Details(feapder.BiddingDetailSpider):
             f_list = detail_info[-1].get('noticeFileBOList')
         except:
             f_list = None
+
         if f_list:
             ff_html = ""
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/', list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/', data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -201,9 +207,12 @@ class Details(feapder.BiddingDetailSpider):
                 ff_html += temp
                 f_type = extract_file_type(f_name, f_url)
                 if f_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=f_name,
+                        file_type=f_type,
+                        download_url=f_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -217,7 +226,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + htxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + htxx_html + ggnr_html + file_html
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
         if file_list:
@@ -226,14 +235,18 @@ class Details(feapder.BiddingDetailSpider):
                 file_url = info.xpath('./@href').extract_first()
                 file_type = extract_file_type(file_name, file_url)
                 if file_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=file_name,
+                        file_type=file_type,
+                        download_url=file_url
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
-        if attachments:
-            list_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        if len(attachments):
+            data_item.projectinfo = {"attachments": attachments}
+
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_gcjs").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gcjs_zbgg/工程建设-合同订立及履约-列表页.py

@@ -10,26 +10,24 @@ import json
 from collections import namedtuple
 
 import feapder
-from items.spider_item import MgpListItem
+from items.spider_item import BidingListItem
 
 from gd_utils import *
 
 
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('工程建设-合同订立及履约', 'a_gdsggzyjypt_gcjs_htdljly', 3),
         ]
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -84,7 +82,7 @@ class Feapder(feapder.BiddingListSpider):
                 "bizCode": f"{tradingProcess}",
                 "projectCode": f"{projectCode}"
             }
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -95,7 +93,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -121,7 +119,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
             yield list_item
 
         # 无限翻页
@@ -130,4 +127,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gcjs", user="1132").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gcjs").start()

gd_gdsggzyjypt_gxhzssyzc/供销合作社社有资产-交易公告-列表页.py

@@ -6,29 +6,29 @@ Created on 2024-09-06
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
+import json
 from collections import namedtuple
+
+import feapder
+from items.spider_item import BidingListItem
+
 from gd_utils import *
-import json
 
 
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('供销合作社社有资产', 'gd_gdsggzyjypt_gxhzssyzc', '3I11', 1),
             Menu('供销合作社社有资产', 'gd_gdsggzyjypt_gxhzssyzc', '3I14', 1),
         ]
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -67,7 +67,6 @@ class Feapder(feapder.BiddingListSpider):
         }
 
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -86,7 +85,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params, proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params, proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -97,7 +96,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()  # 存储数据的管道
+            list_item = BidingListItem()  # 存储数据的管道
             list_item.href = href  # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -123,7 +122,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
             yield list_item
 
         # 无限翻页
@@ -132,4 +130,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gxhzssyzc", user="gdsggzyjypt_gxhzssyzc_jj").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gxhzssyzc").start()

gd_gdsggzyjypt_gxhzssyzc/供销合作社社有资产-详情页.py

@@ -17,23 +17,24 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
+
+    def start_callback(self):
+        self._downloader = AttachmentDownloader()
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=10)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=10)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
         request.proxies = get_proxy(socks5h=True)
-
         request.headers = {
             "Accept": "application/json, text/plain, */*",
             "Accept-Language": "zh-CN,zh;q=0.9",
@@ -48,11 +49,10 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -91,12 +91,13 @@ class Details(feapder.BiddingDetailSpider):
             f_list = detail_info[-1].get('noticeFileBOList')
         except:
             f_list = None
+
         if f_list:
             ff_html = ""
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -113,9 +114,12 @@ class Details(feapder.BiddingDetailSpider):
                 ff_html += temp
                 f_type = extract_file_type(f_name, f_url)
                 if f_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=f_name,
+                        file_type=f_type,
+                        download_url=f_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -129,15 +133,18 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()
 
         fm_type = extract_file_type('公告内容', iframe_url)
         if fm_type:
-            attachmentf = AttachmentDownloader().fetch_attachment(
-                file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+            attachmentf = self._downloader.fetch_attachment(
+                file_name='公告内容',
+                file_type=fm_type,
+                download_url=iframe_url,
+                proxies=request.get_proxies()
+            )
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -147,16 +154,19 @@ class Details(feapder.BiddingDetailSpider):
                 file_url = info.xpath('./@href').extract_first()
                 file_type = extract_file_type(file_name,file_url)
                 if file_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=file_name,
+                        file_type=file_type,
+                        download_url=file_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
-        if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+        if len(attachments) > 0:
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_gxhzssyzc").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gxhzssyzc").start()

gd_gdsggzyjypt_gycq_cjgg/国有产权-交易公告-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('国有产权-交易公告', 'gd_gdsggzyjypt_gycq_cjgg', 10),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -89,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -100,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -127,8 +121,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -137,5 +129,5 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_gycq_jygg", user="1129").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gycq_jygg").start()
 

gd_gdsggzyjypt_gycq_cjgg/国有产权-详情页.py

@@ -17,18 +17,20 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
+
+    def start_callback(self):
+        self._downloader = AttachmentDownloader()
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=30)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=30)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,7 +50,6 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
@@ -113,9 +114,12 @@ class Details(feapder.BiddingDetailSpider):
                 ff_html += temp
                 f_type = extract_file_type(f_name, f_url)
                 if f_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=f_name,
+                        file_type=f_type,
+                        download_url=f_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -135,9 +139,12 @@ class Details(feapder.BiddingDetailSpider):
 
         fm_type = extract_file_type('公告内容', iframe_url)
         if fm_type:
-            attachmentf = AttachmentDownloader().fetch_attachment(
-                file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+            attachmentf = self._downloader.fetch_attachment(
+                file_name='公告内容',
+                file_type=fm_type,
+                download_url=iframe_url,
+                proxies=request.get_proxies()
+            )
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -147,16 +154,19 @@ class Details(feapder.BiddingDetailSpider):
                 file_url = info.xpath('./@href').extract_first()
                 file_type = extract_file_type(file_name,file_url)
                 if file_type:
-                    attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                    attachment = self._downloader.fetch_attachment(
+                        file_name=file_name,
+                        file_type=file_type,
+                        download_url=file_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
-        if attachments:
+        if len(attachments) > 0:
             list_item.projectinfo = {"attachments": attachments}
 
         yield list_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_gycq_jygg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_gycq_jygg").start()

gd_gdsggzyjypt_lqjy_cjgg/林权交易-结果公告-列表页.py

@@ -6,31 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('林权交易-结果公告', 'gd_gdsggzyjypt_lqjy_cjgg', 1),
         ]
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -67,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -88,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -99,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -125,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -135,5 +128,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_lqjy_jggg",user="gdsggzyjypt_lqjy_jggg").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_lqjy_jggg").start()

gd_gdsggzyjypt_ncjtcq/农村集体产权-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-05-13
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('农村集体产权', 'gd_gdsggzyjypt_ncjtcq', 1),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -89,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -100,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -136,5 +130,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_ncjtcq",user="gdsggzyjypt_ncjtcq").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_ncjtcq").start()

gd_gdsggzyjypt_ncjtcq/农村集体产权-详情页.py

@@ -17,18 +17,16 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=10)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,11 +46,10 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -96,7 +93,7 @@ class Details(feapder.BiddingDetailSpider):
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -115,7 +112,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -129,7 +126,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()
 
@@ -137,7 +134,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,14 +146,14 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_ncjtcq").start()
+    Spider(redis_key="lzz:gdsggzyjypt_ncjtcq").start()

gd_gdsggzyjypt_qt_qt/其他交易-交易公告-列表页.py

@@ -6,31 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('其他交易-交易公告', 'gd_gdsggzyjypt_qt_qt', 2),
         ]
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -67,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -88,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -99,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -125,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -135,5 +128,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_qtjy_jygg",user="gdsggzyjypt_qtjy_jygg").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_qtjy_jygg").start()

gd_gdsggzyjypt_qt_qt/其他交易-交易公告-详情页.py

@@ -17,18 +17,17 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=20)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,11 +47,10 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -91,12 +89,13 @@ class Details(feapder.BiddingDetailSpider):
             f_list = detail_info[-1].get('noticeFileBOList')
         except:
             f_list = None
+
         if f_list:
             ff_html = ""
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -114,8 +113,11 @@ class Details(feapder.BiddingDetailSpider):
                 f_type = extract_file_type(f_name, f_url)
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        file_name=f_name,
+                        file_type=f_type,
+                        download_url=f_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -129,15 +131,18 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()
 
         fm_type = extract_file_type('公告内容', iframe_url)
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
-                file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                file_name='公告内容',
+                file_type=fm_type,
+                download_url=iframe_url,
+                proxies=request.get_proxies()
+            )
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -148,15 +153,18 @@ class Details(feapder.BiddingDetailSpider):
                 file_type = extract_file_type(file_name,file_url)
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
-                        file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        file_name=file_name,
+                        file_type=file_type,
+                        download_url=file_url,
+                        proxies=request.get_proxies()
+                    )
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_qtjy_jygg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_qtjy_jygg").start()

gd_gdsggzyjypt_qtjy_jggg/其他交易-结果公告-列表页.py

@@ -6,31 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('其他交易-结果公告', 'gd_gdsggzyjypt_qtjy_jggg', 1),
         ]
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -67,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -88,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -99,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -125,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -135,5 +128,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_qtjy_jggg",user="gdsggzyjypt_qtjy_jggg").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_qtjy_jggg").start()

gd_gdsggzyjypt_qtjy_jggg/其他交易-结果公告-详情页.py

@@ -17,18 +17,17 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=20)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -52,7 +51,7 @@ class Details(feapder.BiddingDetailSpider):
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -96,7 +95,7 @@ class Details(feapder.BiddingDetailSpider):
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -115,7 +114,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -129,7 +128,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()
 
@@ -137,7 +136,7 @@ class Details(feapder.BiddingDetailSpider):
         if fm_type:
             attachmentf = AttachmentDownloader().fetch_attachment(
                 file_name='公告内容', file_type=fm_type, download_url=iframe_url,
-                proxies=request.proxies)
+                proxies=request.get_proxies())
             attachments[str(len(attachments) + 1)] = attachmentf
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
@@ -149,14 +148,14 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_qtjy_jggg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_qtjy_jggg").start()

gd_gdsggzyjypt_tdky_tdcrjh/土地矿业-土地使用权交易公告-详情页.py

@@ -17,18 +17,17 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=30)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=30)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,11 +47,10 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -96,7 +94,7 @@ class Details(feapder.BiddingDetailSpider):
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -115,7 +113,7 @@ class Details(feapder.BiddingDetailSpider):
                 if f_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=f_name, file_type=f_type, download_url=f_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
 
             file_html = f'''
@@ -129,8 +127,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
-
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         file_list = Selector(ggnr_html).xpath('//a[@href]')
         if file_list:
@@ -141,13 +138,14 @@ class Details(feapder.BiddingDetailSpider):
                 if file_type:
                     attachment = AttachmentDownloader().fetch_attachment(
                         file_name=file_name, file_type=file_type, download_url=file_url,
-                        proxies=request.proxies)
+                        proxies=request.get_proxies())
                     attachments[str(len(attachments) + 1)] = attachment
+
         if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_tdky_tdsyqjygg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_tdky_tdsyqjygg").start()

gd_gdsggzyjypt_tdky_tdcrjh/土地矿业-土地出让计划-列表页.py

@@ -6,32 +6,28 @@ Created on 2025-04-09
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('土地矿业-土地出让计划', 'gd_gdsggzyjypt_tdky_tdcrjh', 1),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -89,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -100,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -126,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -136,5 +128,5 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_tdky_tdsyqjygg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_tdky_tdsyqjygg").start()
 

gd_gdsggzyjypt_zfcg_cggg/政府采购-采购公告-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('政府采购-采购公告', 'gd_gdsggzyjypt_zfcg_cggg', 5),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -89,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -100,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -126,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -136,5 +128,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_zfcg_cggg",user="gdsggzyjypt_zfcg_cggg").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_zfcg_cggg").start()

gd_gdsggzyjypt_zfcg_cggg/政府采购-采购公告-详情页.py

@@ -17,18 +17,17 @@ from untils.tools import extract_file_type
 from gd_utils import *
 
 
-class Details(feapder.BiddingDetailSpider):
+class Spider(feapder.BiddingDetailSpider):
 
     def start_requests(self):
-        while True:
-            data_list = self.get_tasks_by_rabbitmq(limit=20)
-            for item in data_list:
-                request_params = item.get("request_params")
-                yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
-                                      deal_detail=item.get("deal_detail"), **request_params,
-                                      callback='parse')
-                
-            break
+        data_list = self.get_tasks(limit=100)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),
+                                  item=item,
+                                  proxies=False,
+                                  deal_detail=item.get("deal_detail"),
+                                  **request_params)
 
     def download_midware(self, request):
         en_str = get_enstr(request.params)
@@ -48,11 +47,10 @@ class Details(feapder.BiddingDetailSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
         attachments = {}
         items = request.item
-        list_item = DataBakItem(**items)
+        data_item = DataBakItem(**items)
 
         detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
         ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]
@@ -96,7 +94,7 @@ class Details(feapder.BiddingDetailSpider):
             index = 1
             for f in f_list:
                 f_id = f.get('rowGuid')
-                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
+                version = "".join(re.findall('new/jygg/(.*?)/',data_item.href))
                 f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                 f_name = f.get('fileName').strip()
                 temp = f'''
@@ -129,7 +127,7 @@ class Details(feapder.BiddingDetailSpider):
         else:
             file_html = ""
 
-        list_item.contenthtml = ggxx_html + ggnr_html + file_html
+        data_item.contenthtml = ggxx_html + ggnr_html + file_html
 
         iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()
 
@@ -153,10 +151,10 @@ class Details(feapder.BiddingDetailSpider):
                     attachments[str(len(attachments) + 1)] = attachment
 
         if attachments:
-            list_item.projectinfo = {"attachments": attachments}
+            data_item.projectinfo = {"attachments": attachments}
 
-        yield list_item
+        yield data_item
 
 
 if __name__ == '__main__':
-    Details(redis_key="lzz:gdsggzyjypt_zfcg_cggg").start()
+    Spider(redis_key="lzz:gdsggzyjypt_zfcg_cggg").start()

gd_gdsggzyjypt_zfcg_gzgg/政府采购-更正公告-列表页.py

@@ -6,32 +6,28 @@ Created on 2024-01-04
 ---------
 @author: lzz
 """
-import feapder
-from items.spider_item import MgpListItem
-from collections import namedtuple
-from gd_utils import *
 import json
+from collections import namedtuple
 
+import feapder
+from items.spider_item import BidingListItem
 
+from gd_utils import *
 
 
-
-class Feapder(feapder.BiddingListSpider):
+class Spider(feapder.BiddingListSpider):
 
     def start_callback(self):
         Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
-
-        self.site = "广东省公共资源交易平台"
-
         self.menus = [
             Menu('政府采购-更正公告', 'gd_gdsggzyjypt_zfcg_gzgg', 3),
         ]
-
+        self.site = "广东省公共资源交易平台"
 
     def start_requests(self):
+        url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
         for menu in self.menus:
-            start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
-            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+            yield feapder.Request(url, item=menu._asdict(), page=1)
 
     def download_midware(self, request):
         page = request.page
@@ -68,9 +64,7 @@ class Feapder(feapder.BiddingListSpider):
             "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
         }
 
-
     def parse(self, request, response):
-
         menu = request.item
         info_list = response.json.get('data').get('pageData')
         for info in info_list:
@@ -89,7 +83,7 @@ class Feapder(feapder.BiddingListSpider):
                 "projectCode": f"{projectCode}"
             }
 
-            nodeId = get_nodeId(params,proxies=request.proxies()).get(noticeId)
+            nodeId = get_nodeId(params,proxies=request.get_proxies()).get(noticeId)
             info['nodeId'] = nodeId
 
             href = create_href(info)
@@ -100,7 +94,7 @@ class Feapder(feapder.BiddingListSpider):
             area = "广东"
             city = regionName
 
-            list_item = MgpListItem()         # 存储数据的管道
+            list_item = BidingListItem()         # 存储数据的管道
             list_item.href = href             # 标书链接
             list_item.unique_key = ('href',)
             list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
@@ -126,8 +120,6 @@ class Feapder(feapder.BiddingListSpider):
             list_item.deal_detail = []  # 抽取正文xpath
             list_item.proxies = False
             list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
-
-
             yield list_item
 
         # 无限翻页
@@ -136,5 +128,4 @@ class Feapder(feapder.BiddingListSpider):
 
 
 if __name__ == "__main__":
-    Feapder(redis_key="lzz:gdsggzyjypt_zfcg_gzgg", user="1123").start()
-
+    Spider(redis_key="lzz:gdsggzyjypt_zfcg_gzgg").start()