lzz_theme

.idea/deployment.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="root@192.168.3.165:8022">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+  </component>
+</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,225 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="CommandLineInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="CythonUsageBeforeDeclarationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyAbstractClassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyArgumentEqualDefaultInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyArgumentListInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAssignmentToLoopOrWithParameterInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAsyncCallInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAttributeOutsideInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAugmentAssignmentInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyBDDParametersInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyBehaveInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyBroadExceptionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyByteLiteralInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCallByClassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCallingNonCallableInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyClassHasNoInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyClassicStyleClassInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyComparisonWithNoneInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCompatibilityInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="ourVersions">
+        <value>
+          <list size="2">
+            <item index="0" class="java.lang.String" itemvalue="2.7" />
+            <item index="1" class="java.lang.String" itemvalue="3.9" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyDataclassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDecoratorInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDefaultArgumentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDeprecationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDictCreationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDictDuplicateKeysInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDocstringTypesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDunderSlotsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyExceptClausesOrderInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyExceptionInheritInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyFinalInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyFromFutureImportInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyGlobalUndefinedInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInconsistentIndentationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyIncorrectDocstringInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInitNewSignatureInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInterpreterInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyListCreationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMandatoryEncodingInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyMethodFirstArgAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodMayBeStaticInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodOverridingInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodParametersInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMissingConstructorInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMissingOrEmptyDocstringInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyMissingTypeHintsInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyNamedTupleInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNestedDecoratorsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNonAsciiCharInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNoneFunctionAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyOldStyleClassesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyOverloadsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="TYPO" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="118">
+            <item index="0" class="java.lang.String" itemvalue="pymongo" />
+            <item index="1" class="java.lang.String" itemvalue="elasticsearch" />
+            <item index="2" class="java.lang.String" itemvalue="apscheduler" />
+            <item index="3" class="java.lang.String" itemvalue="ddddocr" />
+            <item index="4" class="java.lang.String" itemvalue="requests" />
+            <item index="5" class="java.lang.String" itemvalue="urllib3" />
+            <item index="6" class="java.lang.String" itemvalue="redis" />
+            <item index="7" class="java.lang.String" itemvalue="wincertstore" />
+            <item index="8" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="9" class="java.lang.String" itemvalue="aniso8601" />
+            <item index="10" class="java.lang.String" itemvalue="cryptography" />
+            <item index="11" class="java.lang.String" itemvalue="et-xmlfile" />
+            <item index="12" class="java.lang.String" itemvalue="numpy" />
+            <item index="13" class="java.lang.String" itemvalue="click" />
+            <item index="14" class="java.lang.String" itemvalue="onnxruntime" />
+            <item index="15" class="java.lang.String" itemvalue="win32-setctime" />
+            <item index="16" class="java.lang.String" itemvalue="pinyin" />
+            <item index="17" class="java.lang.String" itemvalue="playwright" />
+            <item index="18" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="19" class="java.lang.String" itemvalue="DBUtils" />
+            <item index="20" class="java.lang.String" itemvalue="pycryptodome" />
+            <item index="21" class="java.lang.String" itemvalue="PyMySQL" />
+            <item index="22" class="java.lang.String" itemvalue="itsdangerous" />
+            <item index="23" class="java.lang.String" itemvalue="pyee" />
+            <item index="24" class="java.lang.String" itemvalue="Flask" />
+            <item index="25" class="java.lang.String" itemvalue="openpyxl" />
+            <item index="26" class="java.lang.String" itemvalue="feapder" />
+            <item index="27" class="java.lang.String" itemvalue="emoji" />
+            <item index="28" class="java.lang.String" itemvalue="joblib" />
+            <item index="29" class="java.lang.String" itemvalue="yarg" />
+            <item index="30" class="java.lang.String" itemvalue="nltk" />
+            <item index="31" class="java.lang.String" itemvalue="sgmllib3k" />
+            <item index="32" class="java.lang.String" itemvalue="nbclient" />
+            <item index="33" class="java.lang.String" itemvalue="Wand" />
+            <item index="34" class="java.lang.String" itemvalue="Click" />
+            <item index="35" class="java.lang.String" itemvalue="QtPy" />
+            <item index="36" class="java.lang.String" itemvalue="pycparser" />
+            <item index="37" class="java.lang.String" itemvalue="WsgiDAV" />
+            <item index="38" class="java.lang.String" itemvalue="feedparser" />
+            <item index="39" class="java.lang.String" itemvalue="tinysegmenter" />
+            <item index="40" class="java.lang.String" itemvalue="ftfy" />
+            <item index="41" class="java.lang.String" itemvalue="docopt" />
+            <item index="42" class="java.lang.String" itemvalue="filelock" />
+            <item index="43" class="java.lang.String" itemvalue="PyNaCl" />
+            <item index="44" class="java.lang.String" itemvalue="Js2Py" />
+            <item index="45" class="java.lang.String" itemvalue="pyreadline3" />
+            <item index="46" class="java.lang.String" itemvalue="soupsieve" />
+            <item index="47" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="48" class="java.lang.String" itemvalue="jsonschema" />
+            <item index="49" class="java.lang.String" itemvalue="notebook" />
+            <item index="50" class="java.lang.String" itemvalue="Flask-Caching" />
+            <item index="51" class="java.lang.String" itemvalue="pywin32" />
+            <item index="52" class="java.lang.String" itemvalue="qtconsole" />
+            <item index="53" class="java.lang.String" itemvalue="dnspython" />
+            <item index="54" class="java.lang.String" itemvalue="jieba3k" />
+            <item index="55" class="java.lang.String" itemvalue="pdfminer3k" />
+            <item index="56" class="java.lang.String" itemvalue="kiwisolver" />
+            <item index="57" class="java.lang.String" itemvalue="typing-extensions" />
+            <item index="58" class="java.lang.String" itemvalue="u-msgpack-python" />
+            <item index="59" class="java.lang.String" itemvalue="jupyter-client" />
+            <item index="60" class="java.lang.String" itemvalue="ipykernel" />
+            <item index="61" class="java.lang.String" itemvalue="nbconvert" />
+            <item index="62" class="java.lang.String" itemvalue="APScheduler" />
+            <item index="63" class="java.lang.String" itemvalue="coloredlogs" />
+            <item index="64" class="java.lang.String" itemvalue="pefile" />
+            <item index="65" class="java.lang.String" itemvalue="pyinstaller-hooks-contrib" />
+            <item index="66" class="java.lang.String" itemvalue="fonttools" />
+            <item index="67" class="java.lang.String" itemvalue="jedi" />
+            <item index="68" class="java.lang.String" itemvalue="paramiko" />
+            <item index="69" class="java.lang.String" itemvalue="regex" />
+            <item index="70" class="java.lang.String" itemvalue="platformdirs" />
+            <item index="71" class="java.lang.String" itemvalue="install" />
+            <item index="72" class="java.lang.String" itemvalue="requests-file" />
+            <item index="73" class="java.lang.String" itemvalue="virtualenv" />
+            <item index="74" class="java.lang.String" itemvalue="distlib" />
+            <item index="75" class="java.lang.String" itemvalue="Elixir" />
+            <item index="76" class="java.lang.String" itemvalue="msgpack" />
+            <item index="77" class="java.lang.String" itemvalue="newspaper3k" />
+            <item index="78" class="java.lang.String" itemvalue="rq" />
+            <item index="79" class="java.lang.String" itemvalue="rsa" />
+            <item index="80" class="java.lang.String" itemvalue="pycurl" />
+            <item index="81" class="java.lang.String" itemvalue="json5" />
+            <item index="82" class="java.lang.String" itemvalue="SQLAlchemy" />
+            <item index="83" class="java.lang.String" itemvalue="tblib" />
+            <item index="84" class="java.lang.String" itemvalue="h2" />
+            <item index="85" class="java.lang.String" itemvalue="Flask-Login" />
+            <item index="86" class="java.lang.String" itemvalue="mysql-connector-python" />
+            <item index="87" class="java.lang.String" itemvalue="rq-scheduler" />
+            <item index="88" class="java.lang.String" itemvalue="pyrsistent" />
+            <item index="89" class="java.lang.String" itemvalue="eventlet" />
+            <item index="90" class="java.lang.String" itemvalue="virtualenv-clone" />
+            <item index="91" class="java.lang.String" itemvalue="zipp" />
+            <item index="92" class="java.lang.String" itemvalue="nest-asyncio" />
+            <item index="93" class="java.lang.String" itemvalue="prompt-toolkit" />
+            <item index="94" class="java.lang.String" itemvalue="tldextract" />
+            <item index="95" class="java.lang.String" itemvalue="feedfinder2" />
+            <item index="96" class="java.lang.String" itemvalue="pyinstaller" />
+            <item index="97" class="java.lang.String" itemvalue="croniter" />
+            <item index="98" class="java.lang.String" itemvalue="hyperframe" />
+            <item index="99" class="java.lang.String" itemvalue="Flask-Cors" />
+            <item index="100" class="java.lang.String" itemvalue="Flask-HTTPAuth" />
+            <item index="101" class="java.lang.String" itemvalue="opencv-python" />
+            <item index="102" class="java.lang.String" itemvalue="parso" />
+            <item index="103" class="java.lang.String" itemvalue="tzdata" />
+            <item index="104" class="java.lang.String" itemvalue="ipython" />
+            <item index="105" class="java.lang.String" itemvalue="packaging" />
+            <item index="106" class="java.lang.String" itemvalue="pipreqs" />
+            <item index="107" class="java.lang.String" itemvalue="hpack" />
+            <item index="108" class="java.lang.String" itemvalue="humanfriendly" />
+            <item index="109" class="java.lang.String" itemvalue="pipenv" />
+            <item index="110" class="java.lang.String" itemvalue="tqdm" />
+            <item index="111" class="java.lang.String" itemvalue="Flask-APScheduler" />
+            <item index="112" class="java.lang.String" itemvalue="argon2-cffi" />
+            <item index="113" class="java.lang.String" itemvalue="ply" />
+            <item index="114" class="java.lang.String" itemvalue="Flask-RESTful" />
+            <item index="115" class="java.lang.String" itemvalue="Pillow" />
+            <item index="116" class="java.lang.String" itemvalue="pyjsparser" />
+            <item index="117" class="java.lang.String" itemvalue="requests-toolbelt" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPropertyAccessInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPropertyDefinitionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyProtectedMemberInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyProtocolInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyRedeclarationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyRedundantParenthesesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyReturnFromInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySetFunctionToLiteralInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyShadowingBuiltinsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyShadowingNamesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySimplifyBooleanCheckInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySingleQuotedDocstringInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStatementEffectInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStringExceptionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStringFormatInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStubPackagesAdvertiser" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStubPackagesCompatibilityInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySuperArgumentsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTestParametrizedInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTrailingSemicolonInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTupleAssignmentBalanceInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTupleItemAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypeCheckerInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypeHintsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypedDictInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnboundLocalVariableInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnnecessaryBackslashInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnreachableCodeInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnusedLocalInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/match_spider.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/zbytb" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.8 (FworkSpider_MQ)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.8 (FworkSpider_MQ)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (FworkSpider_MQ)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/match_spider.iml" filepath="$PROJECT_DIR$/.idea/match_spider.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,672 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="BranchesTreeState">
+    <expand>
+      <path>
+        <item name="ROOT" type="e8cecc67:BranchNodeDescriptor" />
+        <item name="LOCAL_ROOT" type="e8cecc67:BranchNodeDescriptor" />
+      </path>
+      <path>
+        <item name="ROOT" type="e8cecc67:BranchNodeDescriptor" />
+        <item name="REMOTE_ROOT" type="e8cecc67:BranchNodeDescriptor" />
+      </path>
+      <path>
+        <item name="ROOT" type="e8cecc67:BranchNodeDescriptor" />
+        <item name="REMOTE_ROOT" type="e8cecc67:BranchNodeDescriptor" />
+        <item name="GROUP_NODE:origin" type="e8cecc67:BranchNodeDescriptor" />
+      </path>
+    </expand>
+    <select />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="1a68edc0-62f4-4dca-9a34-6dbc012edd17" name="Default Changelist" comment="中国招标与采购网爬虫维护">
+      <change beforePath="$PROJECT_DIR$/zbytb/crawler/crawl_scheduler.py" beforeDir="false" afterPath="$PROJECT_DIR$/zbytb/crawler/crawl_scheduler.py" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+        <option value="HTML File" />
+      </list>
+    </option>
+  </component>
+  <component name="FlaskConsoleOptions" custom-start-script="import sys&#10;sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo&#10;locals().update(ScriptInfo(create_app=None).load_app().make_shell_context())&#10;print(&quot;Python %s on %s\nApp: %s [%s]\nInstance: %s&quot; % (sys.version, sys.platform, app.import_name, app.env, app.instance_path))">
+    <envs>
+      <env key="FLASK_APP" value="app" />
+    </envs>
+    <option name="myCustomStartScript" value="import sys&#10;sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo&#10;locals().update(ScriptInfo(create_app=None).load_app().make_shell_context())&#10;print(&quot;Python %s on %s\nApp: %s [%s]\nInstance: %s&quot; % (sys.version, sys.platform, app.import_name, app.env, app.instance_path))" />
+    <option name="myEnvs">
+      <map>
+        <entry key="FLASK_APP" value="app" />
+      </map>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="JupyterTrust" id="48254128-cdcb-4eb9-bfc6-dcdff1f7515d" />
+  <component name="ProjectColorInfo">{
+  &quot;associatedIndex&quot;: 0
+}</component>
+  <component name="ProjectId" id="2IIuXcoSmI9fF8AS9ItoTEu4xng" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent">{
+  &quot;keyToString&quot;: {
+    &quot;DefaultHtmlFileTemplate&quot;: &quot;HTML File&quot;,
+    &quot;Python.ListPageSpider.executor&quot;: &quot;Debug&quot;,
+    &quot;Python.check_utils.executor&quot;: &quot;Run&quot;,
+    &quot;Python.jsl_5s.executor&quot;: &quot;Debug&quot;,
+    &quot;Python.load.executor&quot;: &quot;Debug&quot;,
+    &quot;Python.main.executor&quot;: &quot;Debug&quot;,
+    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    &quot;git-widget-placeholder&quot;: &quot;master&quot;,
+    &quot;last_opened_file_path&quot;: &quot;C:/Users/topnet/Desktop/code/match_spider/zbytb/crawler/clean_html.py&quot;,
+    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
+    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
+    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
+    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
+    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
+    &quot;two.files.diff.last.used.file&quot;: &quot;C:/Users/topnet/Desktop/code/match_spider/zbytb/crawler/clean_html.py&quot;,
+    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  }
+}</component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\topnet\Desktop\code\match_spider\zbytb\crawler" />
+      <recent name="C:\Users\topnet\Desktop\match_spider\zbytb" />
+      <recent name="C:\Users\topnet\Desktop\match_spider\zbytb\utils" />
+    </key>
+  </component>
+  <component name="RunManager" selected="Python.check_utils">
+    <configuration default="true" type="lua.app" factoryName="Lua Application">
+      <option name="program" value="lua.exe" />
+      <option name="file" value="" />
+      <option name="workingDir" value="$PROJECT_DIR$/zbytb" />
+      <option name="debuggerType" value="1" />
+      <option name="params" value="" />
+      <option name="charset" value="UTF-8" />
+      <option name="showConsole" value="true" />
+      <method v="2" />
+    </configuration>
+    <configuration name="ListPageSpider" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="match_spider" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/zbytb/crawler/spiders" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/zbytb/crawler/spiders/ListPageSpider.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="check_utils" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="match_spider" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/zbytb/crawler" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/zbytb/crawler/check_utils.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="jsl_5s" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="match_spider" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/zbytb/crawler" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/zbytb/crawler/jsl_5s.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="load" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="match_spider" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/zbytb/config" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/zbytb/config/load.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="main" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="match_spider" />
+      <option name="ENV_FILES" value="" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/zbytb" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/zbytb/main.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.check_utils" />
+        <item itemvalue="Python.main" />
+        <item itemvalue="Python.ListPageSpider" />
+        <item itemvalue="Python.jsl_5s" />
+        <item itemvalue="Python.load" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SharedIndexes">
+    <attachedChunks>
+      <set>
+        <option value="bundled-js-predefined-1d06a55b98c1-0b3e54e931b4-JavaScript-PY-241.17890.14" />
+        <option value="bundled-python-sdk-5b207ade9991-7e9c3bbb6e34-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-241.17890.14" />
+      </set>
+    </attachedChunks>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="1a68edc0-62f4-4dca-9a34-6dbc012edd17" name="Default Changelist" comment="" />
+      <created>1669881458667</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1669881458667</updated>
+      <workItem from="1669881459814" duration="18000" />
+      <workItem from="1669885286186" duration="251000" />
+      <workItem from="1669887691137" duration="3000" />
+      <workItem from="1669942499033" duration="7000" />
+      <workItem from="1672197559147" duration="1199000" />
+      <workItem from="1672283245272" duration="102000" />
+      <workItem from="1672302308102" duration="839000" />
+      <workItem from="1672303895425" duration="1706000" />
+      <workItem from="1672707294097" duration="259000" />
+      <workItem from="1672800989590" duration="48000" />
+      <workItem from="1674871278302" duration="1403000" />
+      <workItem from="1675241424532" duration="12000" />
+      <workItem from="1675843078448" duration="748000" />
+      <workItem from="1676183392314" duration="343000" />
+      <workItem from="1677820577203" duration="8748000" />
+      <workItem from="1677831279035" duration="3487000" />
+      <workItem from="1677834889723" duration="1229000" />
+      <workItem from="1677893172494" duration="215000" />
+      <workItem from="1678094012256" duration="21000" />
+      <workItem from="1678348013532" duration="137000" />
+      <workItem from="1678759715677" duration="36000" />
+      <workItem from="1679363072700" duration="630000" />
+      <workItem from="1679451696505" duration="53000" />
+      <workItem from="1679480344248" duration="865000" />
+      <workItem from="1680589919691" duration="99000" />
+      <workItem from="1680590624968" duration="218000" />
+      <workItem from="1681284855670" duration="3421000" />
+      <workItem from="1681297804998" duration="2002000" />
+      <workItem from="1681347990851" duration="469000" />
+      <workItem from="1681869613046" duration="326000" />
+      <workItem from="1681952283723" duration="52000" />
+      <workItem from="1681952521061" duration="1966000" />
+      <workItem from="1682487639990" duration="126000" />
+      <workItem from="1683352329940" duration="606000" />
+      <workItem from="1685003680548" duration="651000" />
+      <workItem from="1685085597479" duration="268000" />
+      <workItem from="1685087239873" duration="835000" />
+      <workItem from="1685698146536" duration="14000" />
+      <workItem from="1686037006735" duration="31000" />
+      <workItem from="1686895973580" duration="9545000" />
+      <workItem from="1687137155154" duration="1325000" />
+      <workItem from="1687140458087" duration="836000" />
+      <workItem from="1687158292825" duration="663000" />
+      <workItem from="1688714172808" duration="4751000" />
+      <workItem from="1691633547372" duration="2414000" />
+      <workItem from="1693189346147" duration="32000" />
+      <workItem from="1694164125057" duration="382000" />
+      <workItem from="1694166265616" duration="863000" />
+      <workItem from="1694167899717" duration="1202000" />
+      <workItem from="1694228543726" duration="2369000" />
+      <workItem from="1694240122543" duration="2737000" />
+      <workItem from="1694424344872" duration="874000" />
+      <workItem from="1694502224356" duration="966000" />
+      <workItem from="1694504979539" duration="954000" />
+      <workItem from="1694507453090" duration="280000" />
+      <workItem from="1694575859925" duration="10000" />
+      <workItem from="1694576364627" duration="290000" />
+      <workItem from="1694747229044" duration="21000" />
+      <workItem from="1695628219226" duration="25000" />
+      <workItem from="1695806656319" duration="1259000" />
+      <workItem from="1696922702417" duration="8270000" />
+      <workItem from="1696993875256" duration="1341000" />
+      <workItem from="1696995240032" duration="1879000" />
+      <workItem from="1697076550870" duration="1495000" />
+      <workItem from="1697096812892" duration="894000" />
+      <workItem from="1697174274737" duration="17000" />
+      <workItem from="1697174485667" duration="12000" />
+      <workItem from="1697592232488" duration="58000" />
+      <workItem from="1698395751074" duration="988000" />
+      <workItem from="1698911318184" duration="76000" />
+      <workItem from="1699250385606" duration="2640000" />
+      <workItem from="1699318275974" duration="363000" />
+      <workItem from="1699342241144" duration="549000" />
+      <workItem from="1699580306415" duration="730000" />
+      <workItem from="1699860884925" duration="1878000" />
+      <workItem from="1699923542609" duration="34000" />
+      <workItem from="1699924421703" duration="125000" />
+      <workItem from="1699941856657" duration="612000" />
+      <workItem from="1700031641927" duration="1271000" />
+      <workItem from="1700109388731" duration="69000" />
+      <workItem from="1700127718217" duration="962000" />
+      <workItem from="1700181822517" duration="1245000" />
+      <workItem from="1700719176050" duration="799000" />
+      <workItem from="1700885060587" duration="619000" />
+      <workItem from="1700897923969" duration="576000" />
+      <workItem from="1701054261904" duration="51000" />
+      <workItem from="1701306559900" duration="617000" />
+      <workItem from="1701330012705" duration="6177000" />
+      <workItem from="1701336985094" duration="23000" />
+      <workItem from="1701401599026" duration="96000" />
+      <workItem from="1701401946674" duration="72000" />
+      <workItem from="1701402231345" duration="78000" />
+      <workItem from="1701488331672" duration="50000" />
+      <workItem from="1701996160003" duration="43000" />
+      <workItem from="1701996775315" duration="670000" />
+      <workItem from="1702025921107" duration="25000" />
+      <workItem from="1702026495330" duration="645000" />
+      <workItem from="1702084189570" duration="5192000" />
+      <workItem from="1702263145814" duration="111000" />
+      <workItem from="1702264171825" duration="3040000" />
+      <workItem from="1702271505638" duration="16000" />
+      <workItem from="1702272678778" duration="549000" />
+      <workItem from="1702277823244" duration="1232000" />
+      <workItem from="1702280548848" duration="195000" />
+      <workItem from="1702280920544" duration="132000" />
+      <workItem from="1702631603619" duration="1180000" />
+      <workItem from="1702716234270" duration="3233000" />
+      <workItem from="1703497666365" duration="749000" />
+      <workItem from="1703555794914" duration="255000" />
+      <workItem from="1704182723312" duration="1611000" />
+      <workItem from="1704243069073" duration="773000" />
+      <workItem from="1704253410983" duration="14699000" />
+      <workItem from="1704431970284" duration="163000" />
+      <workItem from="1704694614900" duration="28000" />
+      <workItem from="1706685911504" duration="1357000" />
+      <workItem from="1706836550946" duration="13000" />
+      <workItem from="1706841595090" duration="7402000" />
+      <workItem from="1706861427588" duration="2757000" />
+      <workItem from="1707119371217" duration="655000" />
+      <workItem from="1707125160808" duration="891000" />
+      <workItem from="1708651240026" duration="23000" />
+      <workItem from="1709013440148" duration="788000" />
+      <workItem from="1709015034945" duration="399000" />
+      <workItem from="1709015701660" duration="18000" />
+      <workItem from="1709015895084" duration="157000" />
+      <workItem from="1709111847893" duration="234000" />
+      <workItem from="1709257626947" duration="31000" />
+      <workItem from="1710573568149" duration="411000" />
+      <workItem from="1710574938890" duration="23000" />
+      <workItem from="1710897086729" duration="13000" />
+      <workItem from="1710929411125" duration="1693000" />
+      <workItem from="1710982542472" duration="203000" />
+      <workItem from="1711175897857" duration="73000" />
+      <workItem from="1711356878106" duration="42000" />
+      <workItem from="1711356961781" duration="174000" />
+      <workItem from="1711507647136" duration="1160000" />
+      <workItem from="1712805400429" duration="86000" />
+      <workItem from="1712891389399" duration="164000" />
+      <workItem from="1714961216813" duration="23000" />
+      <workItem from="1716530755696" duration="1425000" />
+      <workItem from="1721712927997" duration="8997000" />
+      <workItem from="1721805137928" duration="224000" />
+      <workItem from="1721811846730" duration="200000" />
+      <workItem from="1728626470367" duration="1418000" />
+      <workItem from="1728959841248" duration="366000" />
+      <workItem from="1732695499571" duration="81000" />
+      <workItem from="1733119661485" duration="15000" />
+      <workItem from="1733120475789" duration="1248000" />
+    </task>
+    <task id="LOCAL-00001" summary="中国招标与采购网数据结构更新">
+      <created>1677829557826</created>
+      <option name="number" value="00001" />
+      <option name="presentableId" value="LOCAL-00001" />
+      <option name="project" value="LOCAL" />
+      <updated>1677829557826</updated>
+    </task>
+    <task id="LOCAL-00002" summary="中国招标与采购网列表页维护">
+      <created>1681286611486</created>
+      <option name="number" value="00002" />
+      <option name="presentableId" value="LOCAL-00002" />
+      <option name="project" value="LOCAL" />
+      <updated>1681286611486</updated>
+    </task>
+    <task id="LOCAL-00003" summary="添加infoformat字段">
+      <created>1681869902739</created>
+      <option name="number" value="00003" />
+      <option name="presentableId" value="LOCAL-00003" />
+      <option name="project" value="LOCAL" />
+      <updated>1681869902739</updated>
+    </task>
+    <task id="LOCAL-00004" summary="拟在建栏目删除infoformat=2">
+      <created>1682487738549</created>
+      <option name="number" value="00004" />
+      <option name="presentableId" value="LOCAL-00004" />
+      <option name="project" value="LOCAL" />
+      <updated>1682487738550</updated>
+    </task>
+    <task id="LOCAL-00005" summary="修改es">
+      <created>1685085850111</created>
+      <option name="number" value="00005" />
+      <option name="presentableId" value="LOCAL-00005" />
+      <option name="project" value="LOCAL" />
+      <updated>1685085850111</updated>
+    </task>
+    <task id="LOCAL-00006" summary="千里马采集二级列表页（市级）">
+      <created>1686905055411</created>
+      <option name="number" value="00006" />
+      <option name="presentableId" value="LOCAL-00006" />
+      <option name="project" value="LOCAL" />
+      <updated>1686905055411</updated>
+    </task>
+    <task id="LOCAL-00007" summary="添加采集成功等待">
+      <created>1687158370138</created>
+      <option name="number" value="00007" />
+      <option name="presentableId" value="LOCAL-00007" />
+      <option name="project" value="LOCAL" />
+      <updated>1687158370138</updated>
+    </task>
+    <task id="LOCAL-00008" summary="中国招标与采购网添加正文包含附件地址处理">
+      <created>1688719143527</created>
+      <option name="number" value="00008" />
+      <option name="presentableId" value="LOCAL-00008" />
+      <option name="project" value="LOCAL" />
+      <updated>1688719143527</updated>
+    </task>
+    <task id="LOCAL-00009" summary="ybw列表页维护">
+      <created>1691635908193</created>
+      <option name="number" value="00009" />
+      <option name="presentableId" value="LOCAL-00009" />
+      <option name="project" value="LOCAL" />
+      <updated>1691635908193</updated>
+    </task>
+    <task id="LOCAL-00010" summary="es过滤条件修改">
+      <created>1694503128304</created>
+      <option name="number" value="00010" />
+      <option name="presentableId" value="LOCAL-00010" />
+      <option name="project" value="LOCAL" />
+      <updated>1694503128304</updated>
+    </task>
+    <task id="LOCAL-00011" summary="清洗title,优化es检索">
+      <created>1696927796499</created>
+      <option name="number" value="00011" />
+      <option name="presentableId" value="LOCAL-00011" />
+      <option name="project" value="LOCAL" />
+      <updated>1696927796499</updated>
+    </task>
+    <task id="LOCAL-00012" summary="update">
+      <created>1696933295890</created>
+      <option name="number" value="00012" />
+      <option name="presentableId" value="LOCAL-00012" />
+      <option name="project" value="LOCAL" />
+      <updated>1696933295890</updated>
+    </task>
+    <task id="LOCAL-00013" summary="update">
+      <created>1696994468506</created>
+      <option name="number" value="00013" />
+      <option name="presentableId" value="LOCAL-00013" />
+      <option name="project" value="LOCAL" />
+      <updated>1696994468506</updated>
+    </task>
+    <task id="LOCAL-00014" summary="ybw更新es查询条件">
+      <created>1699252737819</created>
+      <option name="number" value="00014" />
+      <option name="presentableId" value="LOCAL-00014" />
+      <option name="project" value="LOCAL" />
+      <updated>1699252737819</updated>
+    </task>
+    <task id="LOCAL-00015" summary="竞品更新es 配置">
+      <created>1701331228792</created>
+      <option name="number" value="00015" />
+      <option name="presentableId" value="LOCAL-00015" />
+      <option name="project" value="LOCAL" />
+      <updated>1701331228792</updated>
+    </task>
+    <task id="LOCAL-00016" summary="竞品更新es 配置">
+      <created>1701335718218</created>
+      <option name="number" value="00016" />
+      <option name="presentableId" value="LOCAL-00016" />
+      <option name="project" value="LOCAL" />
+      <updated>1701335718218</updated>
+    </task>
+    <task id="LOCAL-00017" summary="update">
+      <created>1702087634606</created>
+      <option name="number" value="00017" />
+      <option name="presentableId" value="LOCAL-00017" />
+      <option name="project" value="LOCAL" />
+      <updated>1702087634606</updated>
+    </task>
+    <task id="LOCAL-00018" summary="update">
+      <created>1702087896323</created>
+      <option name="number" value="00018" />
+      <option name="presentableId" value="LOCAL-00018" />
+      <option name="project" value="LOCAL" />
+      <updated>1702087896323</updated>
+    </task>
+    <task id="LOCAL-00019" summary="update">
+      <created>1702280704825</created>
+      <option name="number" value="00019" />
+      <option name="presentableId" value="LOCAL-00019" />
+      <option name="project" value="LOCAL" />
+      <updated>1702280704825</updated>
+    </task>
+    <task id="LOCAL-00020" summary="元博网爬虫维护">
+      <created>1704264105664</created>
+      <option name="number" value="00020" />
+      <option name="presentableId" value="LOCAL-00020" />
+      <option name="project" value="LOCAL" />
+      <updated>1704264105664</updated>
+    </task>
+    <task id="LOCAL-00021" summary="中国招标与采购网爬虫维护">
+      <created>1704266735686</created>
+      <option name="number" value="00021" />
+      <option name="presentableId" value="LOCAL-00021" />
+      <option name="project" value="LOCAL" />
+      <updated>1704266735686</updated>
+    </task>
+    <task id="LOCAL-00022" summary="update">
+      <created>1704266935921</created>
+      <option name="number" value="00022" />
+      <option name="presentableId" value="LOCAL-00022" />
+      <option name="project" value="LOCAL" />
+      <updated>1704266935921</updated>
+    </task>
+    <task id="LOCAL-00023" summary="元博网抽取寻源地址">
+      <created>1704269904079</created>
+      <option name="number" value="00023" />
+      <option name="presentableId" value="LOCAL-00023" />
+      <option name="project" value="LOCAL" />
+      <updated>1704269904079</updated>
+    </task>
+    <task id="LOCAL-00024" summary="update">
+      <created>1704432093998</created>
+      <option name="number" value="00024" />
+      <option name="presentableId" value="LOCAL-00024" />
+      <option name="project" value="LOCAL" />
+      <updated>1704432093999</updated>
+    </task>
+    <task id="LOCAL-00025" summary="千里马爬虫维护">
+      <created>1709013824269</created>
+      <option name="number" value="00025" />
+      <option name="presentableId" value="LOCAL-00025" />
+      <option name="project" value="LOCAL" />
+      <updated>1709013824269</updated>
+    </task>
+    <task id="LOCAL-00026" summary="update">
+      <created>1710982740197</created>
+      <option name="number" value="00026" />
+      <option name="presentableId" value="LOCAL-00026" />
+      <option name="project" value="LOCAL" />
+      <updated>1710982740197</updated>
+    </task>
+    <task id="LOCAL-00027" summary="redis配置修改">
+      <created>1711175945173</created>
+      <option name="number" value="00027" />
+      <option name="presentableId" value="LOCAL-00027" />
+      <option name="project" value="LOCAL" />
+      <updated>1711175945174</updated>
+    </task>
+    <task id="LOCAL-00028" summary="中国招标与采购网爬虫维护">
+      <option name="closed" value="true" />
+      <created>1721722900912</created>
+      <option name="number" value="00028" />
+      <option name="presentableId" value="LOCAL-00028" />
+      <option name="project" value="LOCAL" />
+      <updated>1721722900912</updated>
+    </task>
+    <task id="LOCAL-00029" summary="中国招标与采购网爬虫维护">
+      <option name="closed" value="true" />
+      <created>1728627814886</created>
+      <option name="number" value="00029" />
+      <option name="presentableId" value="LOCAL-00029" />
+      <option name="project" value="LOCAL" />
+      <updated>1728627814886</updated>
+    </task>
+    <option name="localTasksCounter" value="30" />
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+  <component name="VcsManagerConfiguration">
+    <MESSAGE value="中国招标与采购网数据结构更新" />
+    <MESSAGE value="中国招标与采购网列表页维护" />
+    <MESSAGE value="添加infoformat字段" />
+    <MESSAGE value="拟在建栏目删除infoformat=2" />
+    <MESSAGE value="修改es" />
+    <MESSAGE value="千里马采集二级列表页（市级）" />
+    <MESSAGE value="添加采集成功等待" />
+    <MESSAGE value="中国招标与采购网添加正文包含附件地址处理" />
+    <MESSAGE value="ybw列表页维护" />
+    <MESSAGE value="es过滤条件修改" />
+    <MESSAGE value="清洗title,优化es检索" />
+    <MESSAGE value="ybw更新es查询条件" />
+    <MESSAGE value="竞品更新es 配置" />
+    <MESSAGE value="元博网爬虫维护" />
+    <MESSAGE value="元博网抽取寻源地址" />
+    <MESSAGE value="千里马爬虫维护" />
+    <MESSAGE value="update" />
+    <MESSAGE value="redis配置修改" />
+    <MESSAGE value="中国招标与采购网爬虫维护" />
+    <option name="LAST_COMMIT_MESSAGE" value="中国招标与采购网爬虫维护" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/ybw/crawler/crawl_scheduler.py</url>
+          <line>55</line>
+          <option name="timeStamp" value="10" />
+        </line-breakpoint>
+      </breakpoints>
+      <default-breakpoints>
+        <breakpoint suspend="NONE" type="python-exception">
+          <properties exception="BaseException" />
+        </breakpoint>
+      </default-breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/match_spider$source_qianlima.coverage" NAME="source_qianlima Coverage Results" MODIFIED="1709013715814" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/qlm" />
+    <SUITE FILE_PATH="coverage/match_spider$detail_spider.coverage" NAME="detail_spider Coverage Results" MODIFIED="1706854034860" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw" />
+    <SUITE FILE_PATH="coverage/match_spider$databases.coverage" NAME="databases Coverage Results" MODIFIED="1702278237140" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw/utils" />
+    <SUITE FILE_PATH="coverage/match_spider$check_utils.coverage" NAME="check_utils Coverage Results" MODIFIED="1728627305533" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/crawler" />
+    <SUITE FILE_PATH="coverage/match_spider$qlm_bc_25.coverage" NAME="qlm_bc_25 Coverage Results" MODIFIED="1702718764737" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/qlm" />
+    <SUITE FILE_PATH="coverage/match_spider$list_spider.coverage" NAME="list_spider Coverage Results" MODIFIED="1696927429903" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw" />
+    <SUITE FILE_PATH="coverage/match_spider$load.coverage" NAME="load Coverage Results" MODIFIED="1721718819313" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/config" />
+    <SUITE FILE_PATH="coverage/match_spider$aa.coverage" NAME="aa Coverage Results" MODIFIED="1707125369903" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw" />
+    <SUITE FILE_PATH="coverage/match_spider$aaa.coverage" NAME="aaa Coverage Results" MODIFIED="1704263373940" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw" />
+    <SUITE FILE_PATH="coverage/match_spider$asd.coverage" NAME="asd Coverage Results" MODIFIED="1677835922790" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw" />
+    <SUITE FILE_PATH="coverage/match_spider$main.coverage" NAME="main Coverage Results" MODIFIED="1721721008930" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb" />
+    <SUITE FILE_PATH="coverage/match_spider$qlm_bc_26.coverage" NAME="qlm_bc_26 Coverage Results" MODIFIED="1702718807584" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/qlm" />
+    <SUITE FILE_PATH="coverage/match_spider$account.coverage" NAME="account Coverage Results" MODIFIED="1702084240495" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw/crawler" />
+    <SUITE FILE_PATH="coverage/match_spider$jsl_5s.coverage" NAME="jsl_5s Coverage Results" MODIFIED="1721719998751" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/crawler" />
+    <SUITE FILE_PATH="coverage/match_spider$ListPageSpider__1_.coverage" NAME="ListPageSpider (1) Coverage Results" MODIFIED="1681285475794" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/crawler/spiders" />
+    <SUITE FILE_PATH="coverage/match_spider$ListPageSpider.coverage" NAME="ListPageSpider Coverage Results" MODIFIED="1721720532940" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/crawler/spiders" />
+    <SUITE FILE_PATH="coverage/match_spider$cccc.coverage" NAME="cccc Coverage Results" MODIFIED="1688717768564" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/zbytb/crawler/spiders" />
+    <SUITE FILE_PATH="coverage/match_spider$title_participle.coverage" NAME="title_participle Coverage Results" MODIFIED="1702278835632" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/ybw/utils" />
+  </component>
+</project>

bdzbw/.idea/bdzbw.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.8 (venv)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="EPYTEXT" />
+    <option name="myDocStringFormat" value="Epytext" />
+  </component>
+</module>

bdzbw/.idea/deployment.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="root@192.168.3.165:8022">
+        <serverdata>
+          <mappings>
+            <mapping local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+  </component>
+</project>

bdzbw/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,225 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="CommandLineInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="CythonUsageBeforeDeclarationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyAbstractClassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyArgumentEqualDefaultInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyArgumentListInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAssignmentToLoopOrWithParameterInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAsyncCallInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAttributeOutsideInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyAugmentAssignmentInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyBDDParametersInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyBehaveInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyBroadExceptionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyByteLiteralInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCallByClassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCallingNonCallableInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyClassHasNoInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyClassicStyleClassInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyComparisonWithNoneInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyCompatibilityInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="ourVersions">
+        <value>
+          <list size="2">
+            <item index="0" class="java.lang.String" itemvalue="2.7" />
+            <item index="1" class="java.lang.String" itemvalue="3.9" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyDataclassInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDecoratorInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDefaultArgumentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDeprecationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDictCreationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDictDuplicateKeysInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDocstringTypesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyDunderSlotsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyExceptClausesOrderInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyExceptionInheritInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyFinalInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyFromFutureImportInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyGlobalUndefinedInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInconsistentIndentationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyIncorrectDocstringInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInitNewSignatureInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyInterpreterInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyListCreationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMandatoryEncodingInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyMethodFirstArgAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodMayBeStaticInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodOverridingInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMethodParametersInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMissingConstructorInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyMissingOrEmptyDocstringInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyMissingTypeHintsInspection" enabled="false" level="TYPO" enabled_by_default="false" />
+    <inspection_tool class="PyNamedTupleInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNestedDecoratorsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNonAsciiCharInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyNoneFunctionAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyOldStyleClassesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyOverloadsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="TYPO" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="118">
+            <item index="0" class="java.lang.String" itemvalue="pymongo" />
+            <item index="1" class="java.lang.String" itemvalue="elasticsearch" />
+            <item index="2" class="java.lang.String" itemvalue="apscheduler" />
+            <item index="3" class="java.lang.String" itemvalue="ddddocr" />
+            <item index="4" class="java.lang.String" itemvalue="requests" />
+            <item index="5" class="java.lang.String" itemvalue="urllib3" />
+            <item index="6" class="java.lang.String" itemvalue="redis" />
+            <item index="7" class="java.lang.String" itemvalue="wincertstore" />
+            <item index="8" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="9" class="java.lang.String" itemvalue="aniso8601" />
+            <item index="10" class="java.lang.String" itemvalue="cryptography" />
+            <item index="11" class="java.lang.String" itemvalue="et-xmlfile" />
+            <item index="12" class="java.lang.String" itemvalue="numpy" />
+            <item index="13" class="java.lang.String" itemvalue="click" />
+            <item index="14" class="java.lang.String" itemvalue="onnxruntime" />
+            <item index="15" class="java.lang.String" itemvalue="win32-setctime" />
+            <item index="16" class="java.lang.String" itemvalue="pinyin" />
+            <item index="17" class="java.lang.String" itemvalue="playwright" />
+            <item index="18" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="19" class="java.lang.String" itemvalue="DBUtils" />
+            <item index="20" class="java.lang.String" itemvalue="pycryptodome" />
+            <item index="21" class="java.lang.String" itemvalue="PyMySQL" />
+            <item index="22" class="java.lang.String" itemvalue="itsdangerous" />
+            <item index="23" class="java.lang.String" itemvalue="pyee" />
+            <item index="24" class="java.lang.String" itemvalue="Flask" />
+            <item index="25" class="java.lang.String" itemvalue="openpyxl" />
+            <item index="26" class="java.lang.String" itemvalue="feapder" />
+            <item index="27" class="java.lang.String" itemvalue="emoji" />
+            <item index="28" class="java.lang.String" itemvalue="joblib" />
+            <item index="29" class="java.lang.String" itemvalue="yarg" />
+            <item index="30" class="java.lang.String" itemvalue="nltk" />
+            <item index="31" class="java.lang.String" itemvalue="sgmllib3k" />
+            <item index="32" class="java.lang.String" itemvalue="nbclient" />
+            <item index="33" class="java.lang.String" itemvalue="Wand" />
+            <item index="34" class="java.lang.String" itemvalue="Click" />
+            <item index="35" class="java.lang.String" itemvalue="QtPy" />
+            <item index="36" class="java.lang.String" itemvalue="pycparser" />
+            <item index="37" class="java.lang.String" itemvalue="WsgiDAV" />
+            <item index="38" class="java.lang.String" itemvalue="feedparser" />
+            <item index="39" class="java.lang.String" itemvalue="tinysegmenter" />
+            <item index="40" class="java.lang.String" itemvalue="ftfy" />
+            <item index="41" class="java.lang.String" itemvalue="docopt" />
+            <item index="42" class="java.lang.String" itemvalue="filelock" />
+            <item index="43" class="java.lang.String" itemvalue="PyNaCl" />
+            <item index="44" class="java.lang.String" itemvalue="Js2Py" />
+            <item index="45" class="java.lang.String" itemvalue="pyreadline3" />
+            <item index="46" class="java.lang.String" itemvalue="soupsieve" />
+            <item index="47" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="48" class="java.lang.String" itemvalue="jsonschema" />
+            <item index="49" class="java.lang.String" itemvalue="notebook" />
+            <item index="50" class="java.lang.String" itemvalue="Flask-Caching" />
+            <item index="51" class="java.lang.String" itemvalue="pywin32" />
+            <item index="52" class="java.lang.String" itemvalue="qtconsole" />
+            <item index="53" class="java.lang.String" itemvalue="dnspython" />
+            <item index="54" class="java.lang.String" itemvalue="jieba3k" />
+            <item index="55" class="java.lang.String" itemvalue="pdfminer3k" />
+            <item index="56" class="java.lang.String" itemvalue="kiwisolver" />
+            <item index="57" class="java.lang.String" itemvalue="typing-extensions" />
+            <item index="58" class="java.lang.String" itemvalue="u-msgpack-python" />
+            <item index="59" class="java.lang.String" itemvalue="jupyter-client" />
+            <item index="60" class="java.lang.String" itemvalue="ipykernel" />
+            <item index="61" class="java.lang.String" itemvalue="nbconvert" />
+            <item index="62" class="java.lang.String" itemvalue="APScheduler" />
+            <item index="63" class="java.lang.String" itemvalue="coloredlogs" />
+            <item index="64" class="java.lang.String" itemvalue="pefile" />
+            <item index="65" class="java.lang.String" itemvalue="pyinstaller-hooks-contrib" />
+            <item index="66" class="java.lang.String" itemvalue="fonttools" />
+            <item index="67" class="java.lang.String" itemvalue="jedi" />
+            <item index="68" class="java.lang.String" itemvalue="paramiko" />
+            <item index="69" class="java.lang.String" itemvalue="regex" />
+            <item index="70" class="java.lang.String" itemvalue="platformdirs" />
+            <item index="71" class="java.lang.String" itemvalue="install" />
+            <item index="72" class="java.lang.String" itemvalue="requests-file" />
+            <item index="73" class="java.lang.String" itemvalue="virtualenv" />
+            <item index="74" class="java.lang.String" itemvalue="distlib" />
+            <item index="75" class="java.lang.String" itemvalue="Elixir" />
+            <item index="76" class="java.lang.String" itemvalue="msgpack" />
+            <item index="77" class="java.lang.String" itemvalue="newspaper3k" />
+            <item index="78" class="java.lang.String" itemvalue="rq" />
+            <item index="79" class="java.lang.String" itemvalue="rsa" />
+            <item index="80" class="java.lang.String" itemvalue="pycurl" />
+            <item index="81" class="java.lang.String" itemvalue="json5" />
+            <item index="82" class="java.lang.String" itemvalue="SQLAlchemy" />
+            <item index="83" class="java.lang.String" itemvalue="tblib" />
+            <item index="84" class="java.lang.String" itemvalue="h2" />
+            <item index="85" class="java.lang.String" itemvalue="Flask-Login" />
+            <item index="86" class="java.lang.String" itemvalue="mysql-connector-python" />
+            <item index="87" class="java.lang.String" itemvalue="rq-scheduler" />
+            <item index="88" class="java.lang.String" itemvalue="pyrsistent" />
+            <item index="89" class="java.lang.String" itemvalue="eventlet" />
+            <item index="90" class="java.lang.String" itemvalue="virtualenv-clone" />
+            <item index="91" class="java.lang.String" itemvalue="zipp" />
+            <item index="92" class="java.lang.String" itemvalue="nest-asyncio" />
+            <item index="93" class="java.lang.String" itemvalue="prompt-toolkit" />
+            <item index="94" class="java.lang.String" itemvalue="tldextract" />
+            <item index="95" class="java.lang.String" itemvalue="feedfinder2" />
+            <item index="96" class="java.lang.String" itemvalue="pyinstaller" />
+            <item index="97" class="java.lang.String" itemvalue="croniter" />
+            <item index="98" class="java.lang.String" itemvalue="hyperframe" />
+            <item index="99" class="java.lang.String" itemvalue="Flask-Cors" />
+            <item index="100" class="java.lang.String" itemvalue="Flask-HTTPAuth" />
+            <item index="101" class="java.lang.String" itemvalue="opencv-python" />
+            <item index="102" class="java.lang.String" itemvalue="parso" />
+            <item index="103" class="java.lang.String" itemvalue="tzdata" />
+            <item index="104" class="java.lang.String" itemvalue="ipython" />
+            <item index="105" class="java.lang.String" itemvalue="packaging" />
+            <item index="106" class="java.lang.String" itemvalue="pipreqs" />
+            <item index="107" class="java.lang.String" itemvalue="hpack" />
+            <item index="108" class="java.lang.String" itemvalue="humanfriendly" />
+            <item index="109" class="java.lang.String" itemvalue="pipenv" />
+            <item index="110" class="java.lang.String" itemvalue="tqdm" />
+            <item index="111" class="java.lang.String" itemvalue="Flask-APScheduler" />
+            <item index="112" class="java.lang.String" itemvalue="argon2-cffi" />
+            <item index="113" class="java.lang.String" itemvalue="ply" />
+            <item index="114" class="java.lang.String" itemvalue="Flask-RESTful" />
+            <item index="115" class="java.lang.String" itemvalue="Pillow" />
+            <item index="116" class="java.lang.String" itemvalue="pyjsparser" />
+            <item index="117" class="java.lang.String" itemvalue="requests-toolbelt" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPropertyAccessInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyPropertyDefinitionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyProtectedMemberInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyProtocolInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyRedeclarationInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyRedundantParenthesesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyReturnFromInitInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySetFunctionToLiteralInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyShadowingBuiltinsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyShadowingNamesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySimplifyBooleanCheckInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySingleQuotedDocstringInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStatementEffectInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStringExceptionInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStringFormatInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStubPackagesAdvertiser" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyStubPackagesCompatibilityInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PySuperArgumentsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTestParametrizedInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTrailingSemicolonInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTupleAssignmentBalanceInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTupleItemAssignmentInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypeCheckerInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypeHintsInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyTypedDictInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnboundLocalVariableInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnnecessaryBackslashInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnreachableCodeInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+    <inspection_tool class="PyUnusedLocalInspection" enabled="true" level="TYPO" enabled_by_default="true" />
+  </profile>
+</component>

bdzbw/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

bdzbw/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (venv)" project-jdk-type="Python SDK" />
+</project>

bdzbw/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/bdzbw.iml" filepath="$PROJECT_DIR$/.idea/bdzbw.iml" />
+    </modules>
+  </component>
+</project>

bdzbw/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

lzz_theme/clgjzbcgjtyxgs/clgj_cookies.txt

@@ -0,0 +1 @@
+eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIyOTYyOTIiLCJpc3MiOiJwbGF0Zm9ybUNlbnRlciIsImlhdCI6MTczNDQ4Mzc0NCwiZXhwIjoxNzM0NTcwMTQ0LCJuYmYiOjE3MzQ0ODM3NDQsImp0aSI6ImE3YzgwZTM0YzllNTQ4ZGRhNzZkYmNlYzQyYWQxNDUxIiwiYXVkIjpbImJVc2VyIl0sInVzZXJJZCI6Mjk2MjkyLCJ1c2VyTmFtZSI6IjEzMjIzMDc0MDAzIiwidGVuYW50SWQiOjIyMDEsImVtYWlsQWRkcmVzcyI6IjEzMjIzMDc0MDAzQGludmFsaWQuY24iLCJ0eXBlIjoiYWNjZXNzVG9rZW4ifQ.ZPkftNcnbszc0A2X3gNRmjkZrtTPjJ6ptQxQm8SmCh3CJ1kNX2bXKKy0QxgKlZIl8OENcbgDugPWXOppZPqxkerX9G3mKzB_9B9_DSBS85lj69yVj0IGMr0dXmsr_-OjKi5nXKgRnRO3dwMr1jaTPh2KXKMCvQZWFh-vwb0Juy8WnXQT_xc3bJTRhQDUN41KUZXyKmvTpVY71LLjTTsvQWGTxXN6mkbKod8yyncEEP-6nhAWllG6g8b6cw59i6KaVxEp_p2UR6dw6RgWWNjK-pAQzx8B0_FLJnwrfcTOqYkIKTEZrCICzzEpia-Lhr7jWLS1Utfmehr8KJtJ8R1pFw

lzz_theme/clgjzbcgjtyxgs/clgjzb_details.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-07-18
+---------
+@summary: 采联国际招标采购集团有限公司 - 详情页
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from utils.attachment import AttachmentDownloader
+from utils.tools import *
+from login_account import Login
+import warnings
+warnings.filterwarnings('ignore')
+
+
+
+class Details:
+
+    def __init__(self):
+        self.db_table = Mongo_client().py_spider
+        self.db_name = self.db_table.theme_list
+        self.zt_details = self.db_table.data_bak
+
+
+    def get_cookies(self):
+        if not os.path.isfile('./clgj_cookies.txt'):
+            Login()
+
+        with open('./clgj_cookies.txt', 'r', encoding='utf-8') as fr:
+            cks = fr.read()
+        return cks
+
+    def detail_get(self, response, item):
+        if "Current user did not login to the application" in response.text:
+            try:
+                os.remove('./clgj_cookies.txt')
+            except:
+                pass
+            return
+        html = response.json().get('content')
+
+        attachments = {}
+
+        file_url = item.get('list_fileurl')
+        file_type = extract_file_type(file_url=file_url)
+        if file_url and file_type:
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=item['title'], file_type=file_type, download_url=file_url)
+            attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            item['projectinfo'] = {"attachments": attachments}
+
+        item["contenthtml"] = html
+
+        item = format_fileds(item)
+
+        try:
+            self.zt_details.insert_one(item)
+            logger.info(f"[采集成功]{item['title']}-{item['publishtime']}")
+        except DuplicateKeyError:
+            logger.info(f"[重复采集]{item['title']}-{item['publishtime']}")
+
+    def fetch_request(self, item):
+        Authorization = self.get_cookies()
+        headers = {
+            "Abp.TenantId": "2201",
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Authorization": f"Bearer {Authorization}",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Origin": "https://qy.choicelink.cn:8301",
+            "Pragma": "no-cache",
+            "Referer": "https://qy.choicelink.cn:8301/",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+        }
+        response = requests.get(url=item.get("parse_url"), headers=headers, timeout=(30, 30), verify=False)
+        time.sleep(1)
+        return response
+
+    def deal_request(self, item):
+        retry_times = 0
+        org_item = item.copy()
+        while retry_times < 5:
+            try:
+                response = self.fetch_request(item)
+                res_code = response.status_code
+                if response and res_code == 200:
+                    self.detail_get(response, item=item)
+                    return True
+                elif res_code == 401:
+                    Login()
+                else:
+                    retry_times += 1
+                    time.sleep(3)
+            except Exception as e:
+                item = org_item
+                logger.error(f"{item.get('competehref')} 采集异常：{e}")
+                retry_times += 1
+                time.sleep(random.randint(3, 6))
+        logger.warning(f"[采集失败]{item.get('competehref')}")
+        return False
+
+    def start(self, limit=1):
+        logger.debug("********** 详情页采集开始 **********")
+
+        with self.db_name.find({"parser_name": "ztpc_clgjzbcgjtyxgs", "is_crawl": False},
+                               sort=[('publishtime', -1)]).limit(limit) as cursor:
+            data_lsit = [dd for dd in cursor]
+        for item in data_lsit:
+            # logger.debug(item)
+            update_id = item["_id"]
+            result = self.deal_request(item)
+            if result is True:
+                self.db_name.update_one({"_id": update_id}, {"$set": {"is_crawl": True}})
+            else:
+                self.db_name.update_one({"_id": update_id}, {"$set": {"failed": True}})
+            time.sleep(random.randint(5, 10))
+
+        logger.debug("********** 详情页采集结束 **********")
+
+
+if __name__ == "__main__":
+    Details().start(limit=20)

lzz_theme/clgjzbcgjtyxgs/clgjzb_list.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-07-18
+---------
+@summary: 采联国际招标采购集团有限公司
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from collections import namedtuple
+from utils.tools import *
+import requests
+from login_account import Login
+import warnings
+warnings.filterwarnings('ignore')
+
+
+class Crawl_Clgj:
+
+    def __init__(self):
+        self.proxy = get_proxy()
+        self.py_spider = Mongo_client().py_spider
+        self.zb_list = self.py_spider.theme_list
+
+        self.r = Redis_client()
+        self.redis_key = 'ztpc_clgjzbcgjtyxgs'
+
+        self.real_cont = 0
+
+    def get_cookies(self):
+        if not os.path.isfile('./clgj_cookies.txt'):
+            Login()
+
+        with open('./clgj_cookies.txt', 'r', encoding='utf-8') as fr:
+            cks = fr.read()
+        return cks
+
+    def fetch_list_page(self, page, menu):
+        logger.debug(f' *** 开始采集第{page}页 ***')
+        Authorization = self.get_cookies()
+        headers = {
+            "Abp.TenantId": "2201",
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Authorization": f"Bearer {Authorization}",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Origin": "https://qy.choicelink.cn:8301",
+            "Pragma": "no-cache",
+            "Referer": "https://qy.choicelink.cn:8301/",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+        }
+
+        url = "https://www.choicelink.cn:8444/cailian/GetAbpArticles"
+        params = {
+            "ext2": menu.tid,
+            "isDispaly": "true",
+            "isInternational": "false",
+            "skipCount": f"{(page-1)*10}"
+        }
+
+        request_params = {
+            "headers": headers,
+            "params": params,
+            "timeout": (30, 30),
+            "verify": False,
+        }
+        resp = requests.get(url, **request_params)
+        time.sleep(1)
+        return resp
+
+    def parser_list_page(self, response, page, menu):
+        if "Current user did not login to the application" in response.text:
+            try:
+                os.remove('./clgj_cookies.txt')
+            except:
+                pass
+            return
+        results_list = []
+        info_list = response.json().get('result').get('items')
+        for info in info_list:
+            hid = info.get('id')
+            projectno = info.get('projectno')
+            endtime = info.get('endtime')
+            fileurl = info.get('fileurl','')
+            title = info.get('title').strip()
+            publish_time = info.get('creationtime').strip()
+            if not endtime or date_to_timestamp(endtime) > int(time.time()) > date_to_timestamp(publish_time):
+                href = f"https://qy.choicelink.cn:8301/project/offlineProject/{menu.cid}?id={hid}&record=%5Bobject%20Object%5D"
+            else:
+                href = f"https://qy.choicelink.cn:8301/project/offlineProject/{menu.cid}?id={hid}&projectNo={projectno}"
+
+            dedup = md5value(title + href + publish_time)
+
+            if not self.r.hexists(self.redis_key, dedup):
+                item = {
+                    "site": "采联国际招标采购集团有限公司",
+                    "channel": menu.channel,
+                    "spidercode": menu.spidercode,
+                    "area": "全国",
+                    "city": "",
+                    "district": "",
+                    "href": "#",
+                    "competehref": href,
+                    "title": title,
+                    "publishtime": publish_time,
+                    "list_fileurl": fileurl,
+                    "parse_url": f"https://www.choicelink.cn:8444/cailian/GetArticlesDataByArticlesId?ArticlesId={hid}",
+                    "parser_name": self.redis_key,
+                    "is_mixed": False,
+                    "is_theme": True,
+                    "retry": 0,
+                    "comeintime": int2long(int(time.time())),
+                    "is_crawl": False,
+                    "failed": False,
+                }
+
+                self.zb_list.insert_one(item)
+                self.r.hset(self.redis_key, dedup, '')
+                results_list.append(item)
+
+        logger.info(f' *** 第{page}页采集完毕 - 共{len(info_list)}条 - 入库{len(results_list)}条 ***')
+
+        return results_list
+
+    def crawl_list_spider(self, page, menu):
+        retry_times = 0
+        while retry_times < 3:
+            try:
+                response = self.fetch_list_page(page=page, menu=menu)
+                res_code = response.status_code
+                logger.debug(f"第{page}页 状态码:{res_code}")
+                if response and res_code == 200:
+                    informations = self.parser_list_page(response=response, page=page, menu=menu)
+                    self.real_cont += len(informations)
+                    logger.info(f"当前已采集 {self.real_cont} 条数据")
+                    time.sleep(random.randint(10,20))
+                    break
+                elif res_code == 401:
+                    Login()
+                else:
+                    retry_times += 1
+                    time.sleep(3)
+            except Exception as e:
+                logger.error(f"第{page}页 采集异常：{e}")
+                retry_times += 1
+                time.sleep(5)
+
+    def start_list(self, menus):
+        logger.debug("********** 列表页开始 **********")
+        for menu in menus:
+            logger.debug(f"++++++ {menu.channel} 开始采集 ++++++")
+            page = menu.crawl_page
+            for page in range(1, page + 1):
+                self.crawl_list_spider(page=page, menu=menu)
+            logger.debug(f"------ {menu.channel} 采集结束 ------")
+
+        logger.debug("********** 列表页结束 **********")
+
+
+if __name__ == '__main__':
+    Menu = namedtuple('Menu', ['channel', 'spidercode', 'tid', 'cid', 'crawl_page'])
+
+    menus = [
+        Menu('招标报名', 'a_clgjzbcgjtyxgs_zbbm', '74166','OfflineSignDetail', 5),
+        Menu('更正公告', 'a_clgjzbcgjtyxgs_gzgg', '74169','CorrectionsNoticeDetail', 2),
+        Menu('结果公告', 'a_clgjzbcgjtyxgs_jggg',  '74168', 'ResultNoticeDetail', 6),
+    ]
+
+    Crawl_Clgj().start_list(menus)

lzz_theme/clgjzbcgjtyxgs/list_start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+ps -ef |grep "clgjzb_list.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 clgjzb_list.py > log/clgjzb_list.out 2>&1 &

lzz_theme/clgjzbcgjtyxgs/log/clgjzb_details.out

@@ -0,0 +1,2 @@
+2024-12-18 15:00:03.882 | DEBUG    | __main__:start:110 - ********** 详情页采集开始 **********
+2024-12-18 15:00:05.268 | DEBUG    | __main__:start:125 - ********** 详情页采集结束 **********

lzz_theme/clgjzbcgjtyxgs/log/clgjzb_list.out

@@ -0,0 +1,64 @@
+2024-12-18 09:00:02.524 | INFO     | utils.tools:get_proxy:112 - 切换代理: {'http': 'socks5://183.147.191.135:8861', 'https': 'socks5://183.147.191.135:8861'}
+2024-12-18 09:00:02.574 | DEBUG    | __main__:start_list:153 - ********** 列表页开始 **********
+2024-12-18 09:00:02.575 | DEBUG    | __main__:start_list:155 - ++++++ 招标报名 开始采集 ++++++
+2024-12-18 09:00:02.575 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第1页 ***
+2024-12-18 09:00:04.510 | DEBUG    | __main__:crawl_list_spider:135 - 第1页 状态码:200
+2024-12-18 09:00:04.791 | INFO     | __main__:parser_list_page:125 -  *** 第1页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:00:04.791 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 10 条数据
+2024-12-18 09:00:18.802 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第2页 ***
+2024-12-18 09:00:20.494 | DEBUG    | __main__:crawl_list_spider:135 - 第2页 状态码:200
+2024-12-18 09:00:20.535 | INFO     | __main__:parser_list_page:125 -  *** 第2页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:00:20.536 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 20 条数据
+2024-12-18 09:00:38.554 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第3页 ***
+2024-12-18 09:00:40.228 | DEBUG    | __main__:crawl_list_spider:135 - 第3页 状态码:200
+2024-12-18 09:00:40.230 | INFO     | __main__:parser_list_page:125 -  *** 第3页采集完毕 - 共10条 - 入库0条 ***
+2024-12-18 09:00:40.231 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 20 条数据
+2024-12-18 09:01:00.251 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第4页 ***
+2024-12-18 09:01:01.873 | DEBUG    | __main__:crawl_list_spider:135 - 第4页 状态码:200
+2024-12-18 09:01:01.886 | INFO     | __main__:parser_list_page:125 -  *** 第4页采集完毕 - 共10条 - 入库1条 ***
+2024-12-18 09:01:01.886 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 21 条数据
+2024-12-18 09:01:14.888 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第5页 ***
+2024-12-18 09:01:16.532 | DEBUG    | __main__:crawl_list_spider:135 - 第5页 状态码:200
+2024-12-18 09:01:16.535 | INFO     | __main__:parser_list_page:125 -  *** 第5页采集完毕 - 共10条 - 入库0条 ***
+2024-12-18 09:01:16.535 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 21 条数据
+2024-12-18 09:01:27.544 | DEBUG    | __main__:start_list:159 - ------ 招标报名 采集结束 ------
+2024-12-18 09:01:27.544 | DEBUG    | __main__:start_list:155 - ++++++ 更正公告 开始采集 ++++++
+2024-12-18 09:01:27.544 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第1页 ***
+2024-12-18 09:01:29.099 | DEBUG    | __main__:crawl_list_spider:135 - 第1页 状态码:200
+2024-12-18 09:01:29.165 | INFO     | __main__:parser_list_page:125 -  *** 第1页采集完毕 - 共10条 - 入库7条 ***
+2024-12-18 09:01:29.166 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 28 条数据
+2024-12-18 09:01:46.180 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第2页 ***
+2024-12-18 09:01:47.767 | DEBUG    | __main__:crawl_list_spider:135 - 第2页 状态码:200
+2024-12-18 09:01:47.781 | INFO     | __main__:parser_list_page:125 -  *** 第2页采集完毕 - 共10条 - 入库0条 ***
+2024-12-18 09:01:47.781 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 28 条数据
+2024-12-18 09:02:00.795 | DEBUG    | __main__:start_list:159 - ------ 更正公告 采集结束 ------
+2024-12-18 09:02:00.795 | DEBUG    | __main__:start_list:155 - ++++++ 结果公告 开始采集 ++++++
+2024-12-18 09:02:00.795 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第1页 ***
+2024-12-18 09:02:02.426 | DEBUG    | __main__:crawl_list_spider:135 - 第1页 状态码:200
+2024-12-18 09:02:02.483 | INFO     | __main__:parser_list_page:125 -  *** 第1页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:02:02.483 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 38 条数据
+2024-12-18 09:02:22.489 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第2页 ***
+2024-12-18 09:02:23.742 | DEBUG    | __main__:crawl_list_spider:135 - 第2页 状态码:401
+2024-12-18 09:02:24.206 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第2页 ***
+2024-12-18 09:02:25.874 | DEBUG    | __main__:crawl_list_spider:135 - 第2页 状态码:200
+2024-12-18 09:02:25.898 | INFO     | __main__:parser_list_page:125 -  *** 第2页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:02:25.898 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 48 条数据
+2024-12-18 09:02:37.903 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第3页 ***
+2024-12-18 09:02:39.552 | DEBUG    | __main__:crawl_list_spider:135 - 第3页 状态码:200
+2024-12-18 09:02:39.598 | INFO     | __main__:parser_list_page:125 -  *** 第3页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:02:39.598 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 58 条数据
+2024-12-18 09:02:53.608 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第4页 ***
+2024-12-18 09:02:55.223 | DEBUG    | __main__:crawl_list_spider:135 - 第4页 状态码:200
+2024-12-18 09:02:55.290 | INFO     | __main__:parser_list_page:125 -  *** 第4页采集完毕 - 共10条 - 入库10条 ***
+2024-12-18 09:02:55.290 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 68 条数据
+2024-12-18 09:03:15.304 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第5页 ***
+2024-12-18 09:03:16.999 | DEBUG    | __main__:crawl_list_spider:135 - 第5页 状态码:200
+2024-12-18 09:03:17.028 | INFO     | __main__:parser_list_page:125 -  *** 第5页采集完毕 - 共10条 - 入库1条 ***
+2024-12-18 09:03:17.028 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 69 条数据
+2024-12-18 09:03:29.038 | DEBUG    | __main__:fetch_list_page:42 -  *** 开始采集第6页 ***
+2024-12-18 09:03:30.683 | DEBUG    | __main__:crawl_list_spider:135 - 第6页 状态码:200
+2024-12-18 09:03:30.686 | INFO     | __main__:parser_list_page:125 -  *** 第6页采集完毕 - 共10条 - 入库0条 ***
+2024-12-18 09:03:30.686 | INFO     | __main__:crawl_list_spider:139 - 当前已采集 69 条数据
+2024-12-18 09:03:40.692 | DEBUG    | __main__:start_list:159 - ------ 结果公告 采集结束 ------
+2024-12-18 09:03:40.692 | DEBUG    | __main__:start_list:161 - ********** 列表页结束 **********
+ >>> 账号：Jianyu2023 登录完成！

lzz_theme/clgjzbcgjtyxgs/login_account.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-07-17
+---------
+@summary: 账密登录
+---------
+@author: Lzz
+"""
+import requests
+
+
+
+def Login(username="Jianyu2023", password="Jianyu@2022!"):
+    session = requests.session()
+
+    headers1 = {
+        "Abp.TenantId": "2",
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/json;charset=UTF-8",
+        "Origin": "https://qy.choicelink.cn:8301",
+        "Pragma": "no-cache",
+        "Referer": "https://qy.choicelink.cn:8301/",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    }
+
+    url1 = "https://middleware.choicelink.cn/security/userCenter/api/TokenAuth/Authenticate"
+    data1 = {
+        "tenant": 2,
+        "password": password,
+        "userNameOrEmailAddress": username
+    }
+    try:
+        res = session.post(url1, headers=headers1, json=data1, timeout=30)
+
+        acctoken = res.json().get('result').get('accessToken')
+
+        headers = {
+            "Abp.TenantId": "2",
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Authorization": f"Bearer {acctoken}",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": "https://qy.choicelink.cn:8301",
+            "Pragma": "no-cache",
+            "Referer": "https://qy.choicelink.cn:8301/",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
+        }
+        url = "https://middleware.choicelink.cn/security/platformCenter/api/TokenAuth/ReloadAuthenticate"
+        data = {
+            "authProvider": 2,
+            "providerKey": 154040,
+            "platformId": 2201
+        }
+        response = session.post(url, headers=headers, json=data, timeout=30)
+
+        hcookies = response.json().get('result').get('accessToken')
+        with open('./clgj_cookies.txt', 'w', encoding='utf-8') as fw:
+            fw.write(str(hcookies))
+
+        print(f" >>> 账号：{username} 登录完成！")
+        return True
+    except Exception as e:
+        print(f" >>> 账号：{username} 登录失败！{e}")
+        return False

lzz_theme/clgjzbcgjtyxgs/start.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# ps -ef |grep "clgjzb_list.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+ps -ef |grep "clgjzb_details.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+# nohup python3 clgjzb_list.py > log/clgjzb_list.out 2>&1 &
+nohup python3 clgjzb_details.py > log/clgjzb_details.out 2>&1 &

lzz_theme/crontab.txt

@@ -0,0 +1,95 @@
+
+NODE_PATH=/usr/lib/node_modules
+
+
+# 竞品采集统计
+20 8 * * * cd /mnt/lzz_monitor/match_jk && ./start.sh
+
+# 重点网站采集统计
+20 8 * * * cd /mnt/lzz_monitor/zdwz_jk && ./start.sh
+
+# 主题爬虫采集统计
+20 8 * * * cd /mnt/lzz_monitor/theme_jk && ./start.sh
+
+# 广东省政府采购网
+*/20 * * * * cd /mnt/lzz_theme/gdszfcgw && ./start.sh
+
+# 湖南省政府采购电子卖场
+10 6-23/3 * * * cd /mnt/lzz_theme/hnszfcgdzmc && ./start.sh
+20 6-23/1 * * * cd /mnt/lzz_theme/hnszfcgdzmc && ./dt_start.sh
+
+# 天津市政府采购网
+*/10 * * * * cd /mnt/lzz_theme/tjszfcgw && ./start.sh
+
+# 香港医院管理局
+0 */3 * * * cd /mnt/lzz_theme/xgyyglj && ./start.sh
+
+# 云南省政府采购网-采购意向公开
+0 */1 * * * cd /mnt/lzz_theme/ynszfcgw && ./det_start.sh
+0 */4 * * * cd /mnt/lzz_theme/ynszfcgw && ./start.sh
+0 6,18 * * * cd /mnt/lzz_theme/ynszfcgw && ./start1.sh
+0 12 * * mon,wed,fri cd /mnt/lzz_theme/ynszfcgw && ./start2.sh
+0 12 * * sun cd /mnt/lzz_theme/ynszfcgw && ./start3.sh
+
+# 优质采（北京）科技发展有限公司
+*/30 7-20 * * * cd /mnt/lzz_theme/yzcbjkjfzyxgs && ./start.sh
+*/30 * * * * cd /mnt/lzz_theme/yzcbjkjfzyxgs && python3 spider_list.py > spider_list.out 2>&1
+*/20 * * * * cd /mnt/lzz_theme/yzcbjkjfzyxgs && python3 sp_list.py > sp_list.out 2>&1
+
+# 中国电子科技集团有限公司电子采购平台
+0 */1 * * * cd /mnt/lzz_theme/zgdzkjjtyxgsdzcgpt && ./list_start.sh
+10 9-18/3 * * * cd /mnt/lzz_theme/zgdzkjjtyxgsdzcgpt && ./start.sh
+0 2 * * * cd /mnt/lzz_theme/zgdzkjjtyxgsdzcgpt && ./ret_start.sh
+
+# 中国五矿集团有限公司
+*/20 * * * * cd /mnt/lzz_theme/zgwkjtyxgs && ./list_start.sh
+*/10 * * * * cd /mnt/lzz_theme/zgwkjtyxgs && ./detail_start.sh
+0 8 */3 * * cd /mnt/lzz_theme/zgwkjtyxgs && ./his_start.sh
+0 7 * * * cd /mnt/lzz_theme/zgwkjtyxgs && ./retry_start.sh
+
+# 中国招标投标公共服务平台
+*/10 6-23 * * * cd /mnt/lzz_theme/zgzbtbggfwpt && ./list_start.sh
+*/10 6-23 * * * cd /mnt/lzz_theme/zgzbtbggfwpt && ./detail_start.sh
+0 6-23/1  * * * cd /mnt/lzz_theme/zgzbtbggfwpt && ./his_start.sh
+0 7 * * * cd /mnt/lzz_theme/zgzbtbggfwpt && ./retry_start.sh
+
+# 全国招标公告公示搜索引擎
+*/20 6-23 * * * cd /mnt/lzz_theme/qgzbgggsssyq && ./start.sh
+10 6-23/2 * * * cd /mnt/lzz_theme/qgzbgggsssyq && python3 py_ssyq_list_bu.py > log/py_ssyq_list_bu.out 2>&1
+20 8 * * * cd /mnt/lzz_theme/qgzbgggsssyq && python3 py_ssyq_details_bu.py > log/py_ssyq_details_bu.out 2>&1
+30 9 * * * cd /mnt/lzz_theme/qgzbgggsssyq && python3 ssyq_main.py > log/ssyq_main.out 2>&1
+
+# 建设项目环境影响登记表备案系统 
+50 8 * * * cd /mnt/lzz_theme/jsxmhjyxdjbbaxt && ./start.sh
+*/20 9-17 * * * cd /mnt/lzz_theme/jsxmhjyxdjbbaxt && ./det_start.sh
+
+# 广东国采招标咨询有限公司
+0 9-17/1 * * * cd /mnt/lzz_theme/gdgczbzxyxgs && ./start.sh
+
+# 采联国际招标采购集团有限公司
+0 10-17/1 * * * cd /mnt/lzz_theme/clgjzbcgjtyxgs && ./start.sh
+0 9 * * * cd /mnt/lzz_theme/clgjzbcgjtyxgs && ./list_start.sh
+
+# 中铁鲁班商务网
+10 7-20/1 * * * cd /mnt/lzz_theme/ztlbsww && ./start.sh
+
+# 中国大唐集团公司电子商务平台
+20 7-20/1 * * * cd /mnt/lzz_theme/zgdtjtgsdzswpt && ./start.sh
+30 9 * * * cd /mnt/lzz_theme/zgdtjtgsdzswpt && python3 retry_dtpy.py > log/retry_dtpy.out 2>&1
+
+# 友云采
+30 7-20/1 * * * cd /mnt/lzz_theme/yyc && ./start.sh
+
+# 施工云采网
+15 7-20/1 * * * cd /mnt/lzz_theme/sgycw && ./start.sh
+
+# 中国移动采购与招标网
+5 7-21/1 * * * cd /mnt/lzz_theme/zgydcgyzbw && ./start.sh
+
+# 甘南州公共资源交易中心
+20 7-21/1 * * * cd /mnt/lzz_theme/gnzggzyjyzx && ./start.sh
+
+# 数智云采云采购平台
+10 6-22/1 * * * cd /mnt/lzz_theme/szycycgpt && ./start.sh
+
+

lzz_theme/gdgczbzxyxgs/gdgczb_cookies.json

@@ -0,0 +1 @@
+{'PHPSESSID': '1fdtf0lfjiq3rbaflqo0rstnu3', 'ZDEDebuggerPresent': 'php,phtml,php3', 'cookie_id_user': '459', 'cookie_sign_user': 'b7e167749e8f8ddac68a927e16e0756a', 'cookie_siteurl_user': 'a04869d4280afae7fe40a770e2263425'}

lzz_theme/gdgczbzxyxgs/gdgczb_details.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-28
+---------
+@summary: 广东国采招标咨询有限公司 - 详情页
+---------
+@author: lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from utils.tools import *
+import requests
+import json
+from login import Login
+import time
+import random
+from parsel import Selector
+
+
+class Details:
+
+    def __init__(self):
+        self.db_table = Mongo_client().py_spider
+        self.db_name = self.db_table.theme_list
+        self.zt_details = self.db_table.data_bak
+        self.redis_key = "ztpc_gdgczbzxyxgs_msg"
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "http://www.zsguocai.com",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+        }
+
+    def get_cookies(self):
+        if not os.path.isfile('./gdgczb_cookies.json'):
+            Login()
+
+        with open('./gdgczb_cookies.json', 'r', encoding='utf-8') as fr:
+            cks = fr.read()
+        ck = json.loads(cks.replace("'", '"'))
+        return ck
+
+    def detail_get(self, response, item):
+        response.encoding = response.apparent_encoding
+        if "详细信息请前往登录系统查看" in response.text:
+            try:
+                os.remove('./gdgczb_cookies.json')
+            except FileNotFoundError:
+                pass
+            logger.warning("登录失效，重新登录！")
+            return
+        root = Selector(text=response.text)
+        html = root.xpath('//div[@class="news_content_detail"]').extract_first()
+        rm_list = ['//div[@id="promore"]']
+
+        pub_time = root.xpath('//div[@id="news_infor"]/text()').extract_first("").strip()
+        if pub_time:
+            item['publishtime'] = pub_time.replace('发布时间：', '')
+        else:
+            item['publishtime'] = get_current_date()
+
+        item['contenthtml'] = remove_htmldata(rm_list, html, root)
+        item = format_fileds(item)
+
+        self.zt_details.insert_one(item)
+        logger.info(f"[采集成功]{item['title']}-{item['publishtime']}")
+
+        return True
+
+    def fetch_request(self, item):
+        cookies = self.get_cookies()
+        response = requests.get(url=item.get("parse_url"), headers=self.headers,
+                                cookies=cookies, timeout=30, verify=False)
+        return response
+
+    def deal_request(self, item):
+
+        retry_times = 0
+        org_item = item.copy()
+        while (retry_times := retry_times + 1) < 3:
+            try:
+                response = self.fetch_request(item)
+                state = response.status_code
+                if response is not None and state == 200:
+                    self.detail_get(response, item)
+                    return True
+                else:
+                    time.sleep(random.randint(5, 10))
+            except Exception as e:
+                item = org_item
+                logger.error(f"{item.get('competehref')} 异常：{e}")
+                time.sleep(8)
+        return False
+
+    def start(self, limit=1):
+        logger.debug("********** 详情页采集开始 **********")
+        count = 0
+        with self.db_name.find({"parser_name": "ztpc_gdgczbzxyxgs",
+                                "failed": False, "is_crawl": False}).limit(limit) as data_lsit:
+            for item in data_lsit:
+                # logger.debug(item)
+                update_id = item["_id"]
+                retry = item["retry"]
+                count += 1
+                if self.deal_request(item):
+                    self.db_name.update_one({"_id": update_id}, {"$set": {"is_crawl": True}})
+                    time.sleep(random.randint(10, 20))
+                else:
+                    retry += 1
+                    self.db_name.update_one({"_id": update_id}, {"$set": {"failed": True, "retry": retry}})
+
+        logger.debug("********** 详情页采集结束 **********")
+
+
+if __name__ == "__main__":
+    Details().start(limit=50)

lzz_theme/gdgczbzxyxgs/gdgczb_list.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-28
+---------
+@summary: 广东国采招标咨询有限公司
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from collections import namedtuple
+from utils.tools import *
+import requests
+from parsel import Selector
+
+
+class Crawl_Gdgczb:
+
+    def __init__(self):
+        self.proxy = get_proxy()
+        self.py_spider = Mongo_client().py_spider
+        self.zb_list = self.py_spider.theme_list
+
+        self.r = Redis_client()
+        self.redis_key = 'ztpc_gdgczbzxyxgs'
+        self.real_cont = 0
+
+    def fetch_list_page(self, page, menu):
+        logger.debug(f' *** 开始采集第{page}页 ***')
+
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "http://www.zsguocai.com/news001.php?id=15&pageid=1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+        }
+
+        url = "http://www.zsguocai.com/news001.php"
+        params = {
+            "id": menu.tid,
+            "pageid": f"{page}"
+        }
+        request_params = {
+            "headers": headers,
+            "params": params,
+            "proxies": self.proxy,
+            "timeout": (30, 30),
+            "verify": False
+        }
+        resp = requests.get(url, **request_params)
+        time.sleep(1)
+        return resp
+
+    def parser_list_page(self, response, page, menu):
+        results_list = []
+        response.encoding = response.apparent_encoding
+        info_list = Selector(response.text).xpath('//div[@class="row abimglist newslist_new"]')
+        for info in info_list:
+            title = info.xpath('./div/a[1]/div[@class="h2tnews"]/text()').extract_first("").strip()
+            href = "http://www.zsguocai.com/" + info.xpath('./div/a[1]/@href').extract_first("").strip()
+            publish_time = ""
+
+            dedup = md5value(href)
+
+            if not self.r.hexists(self.redis_key, dedup):
+                item = {
+                    "site": "广东国采招标咨询有限公司",
+                    "channel": menu.channel,
+                    "spidercode": menu.spidercode,
+                    "area": "广东",
+                    "city": "",
+                    "district": "",
+                    "href": "#",
+                    "competehref": href,
+                    "title": title,
+                    "publishtime": publish_time,
+                    "parse_url": href,
+                    "parser_name": self.redis_key,
+                    "is_mixed": False,
+                    "is_theme": True,
+                    "retry": 0,
+                    "comeintime": int2long(time.time()),
+                    "is_crawl": False,
+                    "failed": False,
+                }
+
+                self.zb_list.insert_one(item)
+                self.r.hset(self.redis_key, dedup, '')
+                results_list.append(item)
+
+        logger.info(f' *** 第{page}页采集完毕 - 共{len(info_list)}条 - 入库{len(results_list)}条 ***')
+
+        return results_list
+
+    def crawl_list_spider(self, page, menu):
+        retry_times = 0
+        while (retry_times := retry_times + 1) < 5:
+            try:
+                response = self.fetch_list_page(page=page, menu=menu)
+                res_code = response.status_code
+                logger.debug(f"第{page}页 状态码:{res_code}")
+                if response is not None and res_code == 200:
+                    informations = self.parser_list_page(response=response, page=page, menu=menu)
+                    self.real_cont += len(informations)
+                    logger.info(f"当前已采集 {self.real_cont} 条数据")
+                    time.sleep(random.random())
+                    break
+                else:
+                    self.proxy = get_proxy()
+                    time.sleep(1)
+            except Exception as e:
+                logger.exception(f"第{page}页 采集异常：{e}")
+                self.proxy = get_proxy()
+                time.sleep(2)
+
+    def start_list(self, menus):
+        logger.debug("********** 列表页开始 **********")
+        for menu in menus:
+            logger.debug(f"++++++ {menu.channel} 开始采集 ++++++")
+            page = menu.crawl_page
+            for page in range(1, page + 1):
+                self.crawl_list_spider(page=page, menu=menu)
+            logger.debug(f"------ {menu.channel} 采集结束 ------")
+
+        logger.debug("********** 列表页结束 **********")
+
+
+if __name__ == '__main__':
+    Menu = namedtuple('Menu', ['channel', 'spidercode', 'tid', 'crawl_page'])
+
+    menus = [
+        Menu('招标公告', 'a_gdgczbzxyxgs_zbgg', '15', 2),
+        Menu('变更公告', 'a_gdgczbzxyxgs_bggg', '16', 1),
+        Menu('结果公示', 'a_gdgczbzxyxgs_jggs', '17', 1),
+    ]
+
+    Crawl_Gdgczb().start_list(menus)

lzz_theme/gdgczbzxyxgs/log/gdgczb_details.out

@@ -0,0 +1,2 @@
+2024-12-18 15:00:03.073 | DEBUG    | __main__:start:102 - ********** 详情页采集开始 **********
+2024-12-18 15:00:03.182 | DEBUG    | __main__:start:118 - ********** 详情页采集结束 **********

lzz_theme/gdgczbzxyxgs/log/gdgczb_list.out

@@ -0,0 +1,25 @@
+2024-12-18 15:00:03.112 | INFO     | utils.tools:get_proxy:112 - 切换代理: {'http': 'socks5://183.154.119.247:8860', 'https': 'socks5://183.154.119.247:8860'}
+2024-12-18 15:00:03.123 | DEBUG    | __main__:start_list:123 - ********** 列表页开始 **********
+2024-12-18 15:00:03.123 | DEBUG    | __main__:start_list:125 - ++++++ 招标公告 开始采集 ++++++
+2024-12-18 15:00:03.124 | DEBUG    | __main__:fetch_list_page:31 -  *** 开始采集第1页 ***
+2024-12-18 15:00:04.961 | DEBUG    | __main__:crawl_list_spider:107 - 第1页 状态码:200
+2024-12-18 15:00:04.977 | INFO     | __main__:parser_list_page:97 -  *** 第1页采集完毕 - 共12条 - 入库0条 ***
+2024-12-18 15:00:04.978 | INFO     | __main__:crawl_list_spider:111 - 当前已采集 0 条数据
+2024-12-18 15:00:05.084 | DEBUG    | __main__:fetch_list_page:31 -  *** 开始采集第2页 ***
+2024-12-18 15:00:06.609 | DEBUG    | __main__:crawl_list_spider:107 - 第2页 状态码:200
+2024-12-18 15:00:06.626 | INFO     | __main__:parser_list_page:97 -  *** 第2页采集完毕 - 共12条 - 入库0条 ***
+2024-12-18 15:00:06.626 | INFO     | __main__:crawl_list_spider:111 - 当前已采集 0 条数据
+2024-12-18 15:00:07.172 | DEBUG    | __main__:start_list:129 - ------ 招标公告 采集结束 ------
+2024-12-18 15:00:07.172 | DEBUG    | __main__:start_list:125 - ++++++ 变更公告 开始采集 ++++++
+2024-12-18 15:00:07.172 | DEBUG    | __main__:fetch_list_page:31 -  *** 开始采集第1页 ***
+2024-12-18 15:00:09.843 | DEBUG    | __main__:crawl_list_spider:107 - 第1页 状态码:200
+2024-12-18 15:00:09.847 | INFO     | __main__:parser_list_page:97 -  *** 第1页采集完毕 - 共12条 - 入库0条 ***
+2024-12-18 15:00:09.847 | INFO     | __main__:crawl_list_spider:111 - 当前已采集 0 条数据
+2024-12-18 15:00:10.753 | DEBUG    | __main__:start_list:129 - ------ 变更公告 采集结束 ------
+2024-12-18 15:00:10.753 | DEBUG    | __main__:start_list:125 - ++++++ 结果公示 开始采集 ++++++
+2024-12-18 15:00:10.753 | DEBUG    | __main__:fetch_list_page:31 -  *** 开始采集第1页 ***
+2024-12-18 15:00:13.638 | DEBUG    | __main__:crawl_list_spider:107 - 第1页 状态码:200
+2024-12-18 15:00:13.641 | INFO     | __main__:parser_list_page:97 -  *** 第1页采集完毕 - 共12条 - 入库0条 ***
+2024-12-18 15:00:13.641 | INFO     | __main__:crawl_list_spider:111 - 当前已采集 0 条数据
+2024-12-18 15:00:14.133 | DEBUG    | __main__:start_list:129 - ------ 结果公示 采集结束 ------
+2024-12-18 15:00:14.133 | DEBUG    | __main__:start_list:131 - ********** 列表页结束 **********

lzz_theme/gdgczbzxyxgs/login.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-28 
+---------
+@summary: 账号登录
+---------
+@author: Lzz
+"""
+import requests
+
+
+def Login(proxies=False):
+    username = "TOP12315000@163.com"
+    print(f" >>> 登录账号：{username} ...")
+
+    headers = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language": "zh-CN,zh;q=0.9",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Origin": "http://www.zsguocai.com",
+        "Pragma": "no-cache",
+        "Referer": "http://www.zsguocai.com/login.php",
+        "Upgrade-Insecure-Requests": "1",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+    }
+    url = "http://www.zsguocai.com/login.php"
+    data = {
+        "rlink": "",
+        "username": "TOP12315000@163.com",
+        "password": "Top00012315"
+    }
+    try:
+        response = requests.post(url, headers=headers, data=data, proxies=proxies, timeout=30, verify=False)
+
+        cookies = response.cookies.get_dict()
+
+        with open('./gdgczb_cookies.json', 'w', encoding='utf-8') as fw:
+            fw.write(str(cookies))
+
+        print(f" >>> 账号：{username} 登录完成！")
+        return True
+    except Exception as e:
+        print(f" >>> 账号：{username} 登录失败！{e}")
+        return False
+

lzz_theme/gdgczbzxyxgs/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+ps -ef |grep "gdgczb_list.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+ps -ef |grep "gdgczb_details.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 gdgczb_list.py > log/gdgczb_list.out 2>&1 &
+nohup python3 gdgczb_details.py > log/gdgczb_details.out 2>&1 &
+

lzz_theme/gdszfcgw/gdszfcgw_daliy_crawl.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-25 
+---------
+@summary: 广东省政府采购网 - 增量采集
+---------
+@author: Lzz
+"""
+from list_spider import Crawl_Gds
+from collections import namedtuple
+import datetime
+
+def get_today_of_day(day_offset=0):
+    return str(datetime.date.today() + datetime.timedelta(days=day_offset))
+
+
+if __name__ == '__main__':
+    Menu = namedtuple('Menu',
+                      ['channel', 'spidercode', 'chan_key', 'noticeType', 'start_day', 'end_day', 'selectTimeName',
+                       'crawl_page'])
+    st = get_today_of_day()
+    ed = get_today_of_day(1)
+    menus = [
+        Menu('首页搜索-采购公告', 'gd_gdszfcgw_syss_cggg', 'fca71be5-fc0c-45db-96af-f513e9abda9d', '', st, ed, 'noticeTime', 50),
+        Menu('首页搜索-电子卖场', 'gd_gdszfcgw_syss_dzmc', '3b49b9ba-48b6-4220-9e8b-eb89f41e9d66', '', st, ed, 'noticeTime', 30),
+        Menu('首页搜索-批量采购', 'gd_gdszfcgw_syss_plcg', 'b893ce4b-616d-4f39-a531-57e958e1475e,751c7726-20a2-47f2-b190-206ae6e9cd89,456ab317-2144-4d8d-b3a7-f26dcecfc096', '', st, ed, 'noticeTime', 1),   # 暂无数据
+        Menu('首页搜索-监管信息', 'gd_gdszfcgw_syss_jgxx', '418093fb-aac8-4562-9be2-97082c101ef7', '', st, ed, 'noticeTime', 1),
+        Menu('首页搜索-采购计划', 'gd_gdszfcgw_syss_cgjh', '95ff31f3-a1af-4bc4-b1a2-54c894476193', '001101', st, ed, 'noticeTime', 50),
+    ]
+    Crawl_Gds().start_list(menus)

lzz_theme/gdszfcgw/gdszfcgw_details_spider.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-04-26
+---------
+@summary: 广东省政府采购网 - 详情页
+---------
+@author: lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from utils.tools import *
+from utils.attachment import AttachmentDownloader
+from threading import Timer
+from utils.clean_html import cleaner
+import requests
+import re
+import time
+import random
+import execjs
+from parsel import Selector
+from collections import namedtuple
+
+
+class Details:
+
+    def __init__(self):
+        self.proxy = get_proxy()
+        self.db_table = Mongo_client().py_spider
+        self.db_name = self.db_table.theme_list
+        self.zt_details = self.db_table.data_bak
+        self.rds = Redis_client()
+        self.redis_key = "ztpc_gdszfcgw_msg"
+        self.delete_key = ""
+        self.end_state = False
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+        }
+
+    def get_html(self, html_source):
+        html_js = "".join(re.findall("var siteIdOriginal='';(.*?)\$\('#info_download'\).hide\(\)", html_source, re.S))
+        if html_js:
+            try:
+                trans = "".join(re.findall(r"openTenderCode.replace(.*?);", html_js))
+                trans_html = html_js.replace(trans, "('\\\\',\"\")")
+                data = "function get_html(){" + trans_html + " return demandAnnouncement }"
+                ctx = execjs.compile(data)
+                html = ctx.call('get_html')
+                return html
+            except:
+                return None
+        else:
+            return None
+
+    def get_file_list(self, html, proxies=False):
+        currPage = "".join(re.findall('var currPage = (.*?);', html))
+        pageSize = "".join(re.findall('var pageSize = (.*?);', html))
+        Id = "".join(re.findall('var currInfoId = "(.*?)"', html))
+        if currPage and pageSize and Id:
+            try:
+                url = f"https://gdgpo.czt.gd.gov.cn/freecms/rest/v1/notice/selectNoticeDocInfo.do?currPage={currPage}&pageSize={pageSize}0&id={Id}"
+                file_res = requests.get(url, headers=self.headers, proxies=proxies, timeout=60, verify=False)
+                return file_res.json().get('data')
+            except:
+                return []
+        else:
+            return []
+
+    def text_search(self, content: str):
+        SearchText = namedtuple('SearchText', ['total'])
+
+        if not content:
+            return SearchText(0)
+
+        results = re.findall('[\u4e00-\u9fa5]', content, re.S)
+        # 列表长度即是中文的字数
+        return SearchText(len(results))
+
+    def detail_get(self, response, item):
+        response.encoding = response.apparent_encoding
+        root = Selector(text=response.text)
+        detail_html = root.xpath('/html/body').extract_first()
+
+        html = ''
+        dxpath_list = ['//div[@id="content"]', '//div[@class="infoCommon"]', '//div[@class="noticeArea"]']
+        for xpath in dxpath_list:
+            html = root.xpath(xpath).extract_first()
+            if html:
+                break
+
+        publishTime = "".join(re.findall('var publishTime = "(.*?)"', response.text))
+        if publishTime:
+            year = root.xpath('//input[@id="year"]').extract_first()
+            month = root.xpath('//input[@id="month"]').extract_first()
+            date = root.xpath('//input[@id="date"]').extract_first()
+            y = publishTime.split(' ')[0].split('-')[0]
+            m = publishTime.split(' ')[0].split('-')[1]
+            d = publishTime.split(' ')[0].split('-')[2]
+            html = html.replace(year, y)
+            html = html.replace(month, m)
+            html = html.replace(date, d)
+
+        js_html = self.get_html(detail_html)
+        if js_html and self.text_search(html).total < 20:
+            html = js_html
+
+        file_name_list = []
+
+        file_list = root.xpath('//a[@href]')
+        attachments = {}
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps']
+        if file_list:
+            for index, info in enumerate(file_list):
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first("").strip()
+                file_type = extract_file_type(file_name, file_url)
+                if file_type and 'http' in file_url:
+                    file_name_list.append(file_name)
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url, proxies=self.proxy)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        js_file_list = self.get_file_list(detail_html, self.proxy)
+        if js_file_list:
+            for infoo in js_file_list:
+                file_name = infoo.get('fileName').strip()
+                file_url = infoo.get('fileUrl').strip()
+                file_type = infoo.get('fileExt').strip()
+
+                if file_type not in file_types:
+                    file_type = file_name.split(".")[-1].lower()
+
+                if file_type in file_types and 'http' in file_url:
+                    file_name_list.append(file_name)
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url, proxies=self.proxy)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            item['projectinfo'] = {"attachments": attachments}
+
+        rm_list = ['//p[contains(@class,"info-title")]','//div[contains(@class,"info-source")]']
+        html = remove_htmldata(rm_list,html,root)
+
+        new_html = html
+        for fn in file_name_list:
+            new_html = new_html.replace(fn, '')
+
+        if self.text_search(new_html).total < 10:
+            html = "详情请访问原网页！"
+
+        item['contenthtml'] = html
+        item = format_fileds(item)
+
+        self.zt_details.insert_one(item)
+        logger.info(f"[采集成功]{item['title']}-{item['publishtime']}")
+
+        return True
+
+    def fetch_request(self, item, proxies=False):
+        response = requests.get(url=item.get("parse_url"), headers=self.headers,
+                                proxies=proxies, timeout=60, verify=False)
+        return response
+
+    def deal_request(self, item):
+
+        response = None
+        retry_times = 0
+        org_item = item.copy()
+        while retry_times < 5:
+            try:
+                response = self.fetch_request(item, self.proxy)
+                state = response.status_code
+                if response is not None and state == 200:
+                    self.detail_get(response, item)
+                    return True
+            except Exception as e:
+                item = org_item
+                logger.error(f"{item['href']} 异常：{e}")
+                time.sleep(3)
+                self.proxy = get_proxy()
+                retry_times += 1
+        return False
+
+    def countSec(self):
+        for count in range(10, 0, -1):
+            print(f'\r{count} 秒 后结束任务', end='')
+            time.sleep(1)
+        print('\r任务结束')
+
+    def de_redis_key(self):
+        self.end_state = True
+        self.rds.hdel(self.redis_key, self.delete_key)
+        logger.warning("当前数据未采集成功，数据已回填！")
+        self.countSec()
+
+    def start(self, limit=1):
+        logger.debug("********** 详情页采集开始 **********")
+        time.sleep(random.random())
+        count = 0
+        ts = Timer(1190, self.de_redis_key)  # 声明一个定时器,设置多少s后执行
+        ts.start()  # 启动定时器
+        with self.db_name.find({"parser_name": "gdszfcgw_zt_details", "failed": False, "is_crawl": False},
+                               no_cursor_timeout=True) as data_lsit:
+            for item in data_lsit:
+                # logger.debug(item)
+                if self.end_state:
+                    break
+                if count >= 200:
+                    break
+                unicode_key = md5value(item.get('href') + item.get('title'))
+                if not self.rds.hexists(self.redis_key, unicode_key):  # 除 动态字段 外所有字段去重
+                    self.rds.hset(self.redis_key, unicode_key, '')
+                    self.delete_key = unicode_key
+                    update_id = item["_id"]
+                    retry = item["retry"]
+                    count += 1
+                    if self.deal_request(item):
+                        self.db_name.update_one({"_id": update_id}, {"$set": {"is_crawl": True}})
+                    else:
+                        retry += 1
+                        self.db_name.update_one({"_id": update_id}, {"$set": {"failed": True, "retry": retry}})
+                        self.rds.hdel(self.redis_key, unicode_key)
+
+        logger.debug("********** 详情页采集结束 **********")
+        ts.cancel()  # 脚本规定时间内正常结束，取消定时器
+
+
+if __name__ == "__main__":
+    Details().start(limit=1)

lzz_theme/gdszfcgw/list_spider.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-25
+---------
+@summary: 广东省政府采购网 - 列表页
+---------
+@author: lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from utils.tools import *
+import time, random
+from urllib.parse import urljoin
+from utils.clean_html import cleaner
+from parsel import Selector
+from utils.attachment import AttachmentDownloader
+from utils.RedisDB import RedisFilter
+from collections import namedtuple
+
+requests.packages.urllib3.disable_warnings()
+requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
+try:
+    requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
+except AttributeError:
+    # no pyopenssl support used / needed / available
+    pass
+
+
+class Crawl_Gds:
+
+    def __init__(self):
+        self.proxy = get_proxy()
+        self.py_spider = Mongo_client().py_spider
+        self.zb_list = self.py_spider.theme_list
+        self.zb_details = self.py_spider.data_bak
+        self.RDS = RedisFilter()
+
+        self.host = 'https://gdgpo.czt.gd.gov.cn'
+        self.real_cont = 0
+        self.con_page = 0
+
+    def get_file(self, html_source, proxies=False):
+        file_list = Selector(text=html_source).xpath('//a[@href]')
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./@title').extract_first() or info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name = file_name.strip()
+                    file_type = extract_file_type(file_name, file_url)
+
+                    if file_type and 'http' in file_url:
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url, proxies=proxies)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        return attachments
+
+    def fetch_list_page(self, page, menu):
+        logger.debug(f' *** 开始采集第{page}页 ***')
+
+        headers = {
+            "Accept": "*/*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=utf-8",
+            "Referer": "https://gdgpo.czt.gd.gov.cn/cms-gd/site/guangdong/qwjsy/index.html?",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+            "nsssjss": nsssjss(),
+            "url": "/freecms/rest/v1/notice/selectInfoMoreChannel.do"
+        }
+        url = "https://gdgpo.czt.gd.gov.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do"
+        params = {
+            "siteId": "cd64e06a-21a7-4620-aebc-0576bab7e07a",
+            "channel": menu.chan_key,
+            "title": "",
+            "content": "",
+            "regionCode": "",
+            "noticeType": menu.noticeType,
+            "operationStartTime": f"{menu.start_day} 00:00:00",
+            "operationEndTime": f"{menu.end_day} 00:00:00",
+            "selectTimeName": menu.selectTimeName,
+            "purchaser": "",
+            "agency": "",
+            "currPage": f"{page}",
+            "pageSize": "15",
+            "verifyCode": down_load_image(self.proxy)
+        }
+
+        request_params = {
+            "headers": headers,
+            "params": params,
+            "proxies": self.proxy,
+            "timeout": 30,
+            "verify": False,
+        }
+        resp = requests.get(url, **request_params)
+        time.sleep(1)
+        return resp
+
+    def parser_list_page(self, response, page, menu):
+        results_list = []
+        results_detail = []
+        info_list = response.json().get("data")
+
+        if type(info_list) == list:
+            for info in info_list:
+                href = info.get("pageurl")
+                title = info.get("shorttitle")
+                create_time = info.get("noticeTime")
+                href = urljoin(self.host, href)
+                content = info.get("content")
+                dedup = [title,href,create_time]
+                if not self.RDS.data_filter(dedup):
+                    if not content or len(content.strip()) < 5:
+                        item = {
+                            "title": title,
+                            "publishtime": create_time,
+                            "spidercode": menu.spidercode,
+                            "site": "广东省政府采购网",
+                            "channel": menu.channel,
+                            "area": "广东",
+                            "city": "",
+                            "href": href,
+                            "publishdept": "",
+                            "iscompete": True,
+                            "type": "",
+                            "T": "bidding",
+                            "infoformat": 1,
+                            "l_np_publishtime": "",
+                            "is_mixed": False,
+                            "is_theme": True,
+                            "is_crawl": False,
+                            "failed": False,
+                            "retry": 0,
+                            "comeintime": int2long(time.time()),
+                            "sendflag": "false",
+                            "_d": "comeintime",
+                            "parser_name": "gdszfcgw_zt_details",
+                            "parse_url": href,
+                        }
+
+                        results_list.append(item)
+                        self.zb_list.insert_one(item)
+                    else:
+                        pub_time = handle_publish_time(create_time)
+                        item = {
+                            "title": title,
+                            "s_title": title,
+                            "publishtime": pub_time[0],
+                            "spidercode": menu.spidercode,
+                            "site": "广东省政府采购网",
+                            "channel": menu.channel,
+                            "area": "广东",
+                            "city": "",
+                            "href": href,
+                            "publishdept": "",
+                            "iscompete": True,
+                            "type": "",
+                            "T": "bidding",
+                            "infoformat": 1,
+                            "comeintime": int2long(time.time()),
+                            "l_np_publishtime": pub_time[1],
+                            "is_mixed": True,
+                            "is_theme": True,
+                            "sendflag": "false",
+                            "_d": "comeintime",
+                            "contenthtml": content,
+                            "detail": cleaner(html=content, special=None, completely=False),
+                        }
+                        attachments = self.get_file(content)
+                        if attachments:
+                            item['projectinfo'] = {"attachments": attachments}
+
+                        results_detail.append(item)
+                        self.zb_details.insert_one(item)
+
+                    self.RDS.data_save_redis(dedup)
+                else:
+                    logger.warning(f"[重复数据] {href}")
+
+            logger.info(f' *** 第{page}页采集完毕 - {len(info_list)} - 入库 {len(results_list) + len(results_detail)} 条***')
+        else:
+            raise ValueError("请求失败！")
+        return results_list, results_detail
+
+    def crawl_list_spider(self, page, menu):
+        if page % 100 == 0:
+            self.proxy = get_proxy()
+        response = None
+        retry_times = 0
+        while retry_times < 3:
+            try:
+                response = self.fetch_list_page(page, menu)
+                logger.debug(f"第{page}页 状态:{response.status_code}")
+                if response is not None and response.status_code == 200:
+                    informations = self.parser_list_page(response, page, menu)
+                    crawl_num = len(informations[0]) + len(informations[1])
+                    self.real_cont += crawl_num
+                    if crawl_num == 0:
+                        self.con_page += 1
+                    else:
+                        self.con_page == 0
+                    time.sleep(random.random())
+                break
+            except Exception as e:
+                logger.error(f"第{page}页 异常：{e}")
+                time.sleep(3)
+                self.proxy = get_proxy()
+                retry_times += 1
+
+    def start_list(self, menus):
+        logger.info("********** 列表页开始 **********")
+        for menu in menus:
+            logger.debug(f"++++++ {menu.channel} 开始采集 ++++++")
+            page = menu.crawl_page
+            for page in range(1, page + 1):
+                self.crawl_list_spider(page=page, menu=menu)
+                if self.con_page >= 20:
+                    logger.warning(f"连续 < {self.con_page} > 页无数据，本轮采集结束！")
+                    self.con_page = 0
+                    break
+            logger.info(f"当前已采集 {self.real_cont} 条数据")
+            logger.debug(f"------ {menu.channel} 采集结束 ------")
+
+        logger.info("********** 列表页结束 **********")
+

lzz_theme/gdszfcgw/log/gdszfcgw_details_spider.out

@@ -0,0 +1,135 @@
+2024-12-18 15:40:02.635 | INFO     | utils.tools:get_proxy:112 - 切换代理: {'http': 'socks5://125.82.77.117:8860', 'https': 'socks5://125.82.77.117:8860'}
+2024-12-18 15:40:02.636 | DEBUG    | __main__:start:206 - ********** 详情页采集开始 **********
+2024-12-18 15:40:24.032 | INFO     | __main__:detail_get:164 - [采集成功]珠海市华中师范大学(珠海）附属中学实验楼109室天棚粉刷等零星工程-2024-12-18 15:17:05
+2024-12-18 15:40:24.718 | INFO     | __main__:detail_get:164 - [采集成功]采购A4喷墨机-2024-12-18 15:17:05
+2024-12-18 15:40:25.495 | INFO     | __main__:detail_get:164 - [采集成功]院内家具采购-2024-12-18 15:17:00
+2024-12-18 15:40:26.094 | INFO     | __main__:detail_get:164 - [采集成功]珠海市金湾区华丰小学采购六层双面图书架-2024-12-18 15:16:51
+2024-12-18 15:40:26.700 | INFO     | __main__:detail_get:164 - [采集成功]退役军人服务站台式计算机-2024-12-18 15:16:50
+2024-12-18 15:40:27.416 | INFO     | __main__:detail_get:164 - [采集成功]高清鼻内镜成像系统及高清电子消化道内镜系统-2024-12-18 15:16:15
+2024-12-18 15:40:28.005 | INFO     | __main__:detail_get:164 - [采集成功]零星家具采购-2024-12-18 15:16:10
+2024-12-18 15:40:28.718 | INFO     | __main__:detail_get:164 - [采集成功]2024年学生课桌椅（2）-2024-12-18 15:15:45
+2024-12-18 15:40:29.414 | INFO     | __main__:detail_get:164 - [采集成功]2025年公务用车定点加油-2024-12-18 15:15:45
+2024-12-18 15:40:30.046 | INFO     | __main__:detail_get:164 - [采集成功]洗手间清洁服务-2024-12-18 15:15:35
+2024-12-18 15:40:30.634 | INFO     | __main__:detail_get:164 - [采集成功]12月份宣传制作费用-2024-12-18 15:15:30
+2024-12-18 15:40:31.327 | INFO     | __main__:detail_get:164 - [采集成功]2024年防范和处置非法集资培训手册-2024-12-18 15:15:15
+2024-12-18 15:40:31.945 | INFO     | __main__:detail_get:164 - [采集成功]揭阳日报社2025年度绩效工资方案和广告发行责任书印刷服务采购-2024-12-18 15:15:05
+2024-12-18 15:40:32.552 | INFO     | __main__:detail_get:164 - [采集成功]佛山市禅城区人民法院2024年报废资产评估服务采购-2024-12-18 15:15:05
+2024-12-18 15:40:33.199 | INFO     | __main__:detail_get:164 - [采集成功]房产评估2-2024-12-18 15:15:00
+2024-12-18 15:40:33.787 | INFO     | __main__:detail_get:164 - [采集成功]职工书屋改造工程-2024-12-18 15:15:00
+2024-12-18 15:40:34.395 | INFO     | __main__:detail_get:164 - [采集成功]广州市医疗保险服务中心白云分中心2025年办公场地物业管理服务项目-2024-12-18 15:14:45
+2024-12-18 15:40:35.129 | INFO     | __main__:detail_get:164 - [采集成功]揭阳日报社物业管理服务（保洁）采购-2024-12-18 15:14:20
+2024-12-18 15:40:35.733 | INFO     | __main__:detail_get:164 - [采集成功]办公家具-2024-12-18 15:14:15
+2024-12-18 15:40:36.513 | INFO     | __main__:detail_get:164 - [采集成功]广州城市职业学院资产综合管理系统维护-2024-12-18 15:14:15
+2024-12-18 15:40:37.202 | INFO     | __main__:detail_get:164 - [采集成功]财务股采购复印纸-2024-12-18 15:14:10
+2024-12-18 15:40:38.412 | INFO     | __main__:detail_get:164 - [采集成功]采购多功能一体机-2024-12-18 15:14:05
+2024-12-18 15:40:39.316 | INFO     | __main__:detail_get:164 - [采集成功]复印机采购项目-2024-12-18 15:13:40
+2024-12-18 15:40:39.870 | INFO     | __main__:detail_get:164 - [采集成功]文娱室升级改造-2024-12-18 15:13:30
+2024-12-18 15:40:40.522 | INFO     | __main__:detail_get:164 - [采集成功]多功能一体机采购项目-2024-12-18 15:13:15
+2024-12-18 15:40:41.101 | INFO     | __main__:detail_get:164 - [采集成功]西樵镇高质量推进城镇生活污水治理项目（西樵山零星污水收集工程）-2024-12-18 15:13:15
+2024-12-18 15:40:41.675 | INFO     | __main__:detail_get:164 - [采集成功]广州市发展和改革委员会价格认定中心档案服务-2024-12-18 15:13:00
+2024-12-18 15:40:42.363 | INFO     | __main__:detail_get:164 - [采集成功]多功能一体机采购项目-2024-12-18 15:13:00
+2024-12-18 15:40:42.942 | INFO     | __main__:detail_get:164 - [采集成功]采购台式计算机-2024-12-18 15:12:50
+2024-12-18 15:40:43.650 | INFO     | __main__:detail_get:164 - [采集成功]江门市民政局复印纸定点采购项目-2024-12-18 15:32:45
+2024-12-18 15:40:44.361 | INFO     | __main__:detail_get:164 - [采集成功]2025年小谷围街北亭村、南亭村、穗石村、 星汇居委综治中心综治视联网会议系统 通信链路及服务费用项目-2024-12-18 15:32:45
+2024-12-18 15:40:45.098 | INFO     | __main__:detail_get:164 - [采集成功]复印纸采购-2024-12-18 15:32:30
+2024-12-18 15:40:45.708 | INFO     | __main__:detail_get:164 - [采集成功]潮州市广播电视台2024年度印刷品采购服务-2024-12-18 15:32:30
+2024-12-18 15:40:46.325 | INFO     | __main__:detail_get:164 - [采集成功]A4纸-2024-12-18 15:32:20
+2024-12-18 15:40:47.029 | INFO     | __main__:detail_get:164 - [采集成功]佛山市教师发展中心打印机等办公设备采购-2024-12-18 15:32:05
+2024-12-18 15:40:47.993 | INFO     | __main__:detail_get:164 - [采集成功]印刷明信片、美食地图-2024-12-18 15:31:55
+2024-12-18 15:40:48.692 | INFO     | __main__:detail_get:164 - [采集成功]西门派出所对李某涉嫌诈骗案审计服务-2024-12-18 15:31:40
+2024-12-18 15:40:49.290 | INFO     | __main__:detail_get:164 - [采集成功]中共大埔县委办公室物业管理服务-2024-12-18 15:31:31
+2024-12-18 15:40:49.899 | INFO     | __main__:detail_get:164 - [采集成功]办公家具-2024-12-18 15:31:30
+2024-12-18 15:40:50.495 | INFO     | __main__:detail_get:164 - [采集成功]《阳春古建筑》印刷-2024-12-18 15:31:05
+2024-12-18 15:40:51.063 | INFO     | __main__:detail_get:164 - [采集成功]印刷服务-2024-12-18 15:31:05
+2024-12-18 15:40:51.628 | INFO     | __main__:detail_get:164 - [采集成功]购复印纸-2024-12-18 15:30:50
+2024-12-18 15:40:52.244 | INFO     | __main__:detail_get:164 - [采集成功]复印纸-2024-12-18 15:30:50
+2024-12-18 15:40:52.946 | INFO     | __main__:detail_get:164 - [采集成功]购买A4复印纸28箱-2024-12-18 15:30:40
+2024-12-18 15:40:54.646 | INFO     | __main__:detail_get:164 - [采集成功]黄石街道黄石花园第一社区采购家具一批-2024-12-18 15:30:40
+2024-12-18 15:40:55.373 | INFO     | __main__:detail_get:164 - [采集成功]复印纸-2024-12-18 15:30:40
+2024-12-18 15:40:55.992 | INFO     | __main__:detail_get:164 - [采集成功]东莞市中堂镇专职消防队政务专网信息传输服务-2024-12-18 15:30:35
+2024-12-18 15:40:56.602 | INFO     | __main__:detail_get:164 - [采集成功]黄石街道黄石花园第一社区采购电子设备一批-2024-12-18 15:30:25
+2024-12-18 15:40:57.198 | INFO     | __main__:detail_get:164 - [采集成功]始兴分站电脑软件采购项目-2024-12-18 15:30:01
+2024-12-18 15:40:57.845 | INFO     | __main__:detail_get:164 - [采集成功]采购台式计算机电脑一批-2024-12-18 15:30:00
+2024-12-18 15:40:58.507 | INFO     | __main__:detail_get:164 - [采集成功]刘本章小学 2024-2025学年度第一学期三至六年级试卷订购-2024-12-18 15:29:35
+2024-12-18 15:40:59.084 | INFO     | __main__:detail_get:164 - [采集成功]办公用品（A3打印纸）-2024-12-18 15:29:25
+2024-12-18 15:40:59.660 | INFO     | __main__:detail_get:164 - [采集成功]博罗县人民检察院台式计算机主机采购计划-2024-12-18 15:29:15
+2024-12-18 15:41:00.656 | INFO     | __main__:detail_get:164 - [采集成功]A4复印纸采购项目-2024-12-18 15:29:15
+2024-12-18 15:41:01.292 | INFO     | __main__:detail_get:164 - [采集成功]2024年“基础教育精品课”市优精品课证书制作-2024-12-18 15:28:45
+2024-12-18 15:41:01.902 | INFO     | __main__:detail_get:164 - [采集成功]表单印刷-2024-12-18 15:28:35
+2024-12-18 15:41:02.534 | INFO     | __main__:detail_get:164 - [采集成功]关于公服办采购惠托育服务点家具的项目-2024-12-18 15:28:35
+2024-12-18 15:41:03.149 | INFO     | __main__:detail_get:164 - [采集成功]采购 希沃SFV86EV触控一体机,-2024-12-18 15:28:30
+2024-12-18 15:41:03.766 | INFO     | __main__:detail_get:164 - [采集成功]定点采购汕头市潮师高级中学校园设施加固维护-2024-12-18 15:28:25
+2024-12-18 15:41:04.608 | INFO     | __main__:detail_get:164 - [采集成功]采购视频会议设备教育套装-2024-12-18 15:28:20
+2024-12-18 15:41:05.182 | INFO     | __main__:detail_get:164 - [采集成功]2024年珠海市白蕉镇卫生院申购印刷类一批-2024-12-18 15:28:05
+2024-12-18 15:41:05.884 | INFO     | __main__:detail_get:164 - [采集成功]打印机及复印纸采购-2024-12-18 15:28:05
+2024-12-18 15:41:06.510 | INFO     | __main__:detail_get:164 - [采集成功]鹤山市医保中心印制医保经办业务表格一批-2024-12-18 15:28:01
+2024-12-18 15:41:07.108 | INFO     | __main__:detail_get:164 - [采集成功]2025年后勤综合服务采购项目-2024-12-18 15:28:01
+2024-12-18 15:41:07.906 | INFO     | __main__:detail_get:164 - [采集成功]铺埔儒林第--中共潮澄饶中心县委机关旧址修缮及布展项目-2024-12-18 15:28:00
+2024-12-18 15:41:08.486 | INFO     | __main__:detail_get:164 - [采集成功]采购增班办公家具-2024-12-18 15:28:00
+2024-12-18 15:41:09.070 | INFO     | __main__:detail_get:164 - [采集成功]定点采购拆除教学楼铁窗罩及更改教室铁窗罩等服务-2024-12-18 15:27:55
+2024-12-18 15:41:09.710 | INFO     | __main__:detail_get:164 - [采集成功]2024-2025学年第一学期期末试卷-2024-12-18 15:27:40
+2024-12-18 15:41:10.340 | INFO     | __main__:detail_get:164 - [采集成功]广东职业技术学院禅城校区部分校园道路铺装沥青工程-2024-12-18 15:27:25
+2024-12-18 15:41:11.037 | INFO     | __main__:detail_get:164 - [采集成功]党政和人大办公室购买龙穴街道市场监督管理所前台-2024-12-18 15:27:25
+2024-12-18 15:41:11.726 | INFO     | __main__:detail_get:164 - [采集成功]定点采购汕头市潮师高级中学学生宿舍、教学楼窗罩改逃生窗-2024-12-18 15:27:20
+2024-12-18 15:41:13.052 | INFO     | __main__:detail_get:164 - [采集成功]防冲撞硬质隔离设施建设-2024-12-18 15:27:20
+2024-12-18 15:41:13.634 | INFO     | __main__:detail_get:164 - [采集成功]2024年下期11月份资料印刷费-2024-12-18 15:27:11
+2024-12-18 15:41:14.244 | INFO     | __main__:detail_get:164 - [采集成功]鹤山市医保中心印制医保经办业务表格一批-2024-12-18 15:27:10
+2024-12-18 15:41:14.899 | INFO     | __main__:detail_get:164 - [采集成功]江东新区政务服务中心多功能办公用纸（A4复印纸）购置-2024-12-18 15:27:00
+2024-12-18 15:41:15.497 | INFO     | __main__:detail_get:164 - [采集成功]广州城市职业学院智慧后勤信息管理系统车辆管理模块功能升级-2024-12-18 15:26:45
+2024-12-18 15:41:17.126 | INFO     | __main__:detail_get:164 - [采集成功]定点采购汕头市潮师高级中学宿舍1-6栋设施拆窗服务-2024-12-18 15:26:45
+2024-12-18 15:41:19.191 | INFO     | __main__:detail_get:164 - [采集成功]【定点议价】东莞第一法院采购中堂法庭安检室不锈钢拉闸门工程-2024-12-18 15:26:36
+2024-12-18 15:41:19.990 | INFO     | __main__:detail_get:164 - [采集成功]办公电脑采购-2024-12-18 15:26:25
+2024-12-18 15:41:21.422 | INFO     | __main__:detail_get:164 - [采集成功]东莞市长安中学运动场力量房-2024-12-18 15:26:20
+2024-12-18 15:41:22.108 | INFO     | __main__:detail_get:164 - [采集成功]印刷服务-2024-12-18 15:26:20
+2024-12-18 15:41:22.697 | INFO     | __main__:detail_get:164 - [采集成功]打印机购买-2024-12-18 15:26:00
+2024-12-18 15:41:23.323 | INFO     | __main__:detail_get:164 - [采集成功]电脑购买-2024-12-18 15:25:50
+2024-12-18 15:41:24.575 | INFO     | __main__:detail_get:164 - [采集成功]江海街敦和路片区综合治理智慧安防建设项目-2024-12-18 15:25:50
+2024-12-18 15:41:25.222 | INFO     | __main__:detail_get:164 - [采集成功]4个文件柜和1个电子密码保险箱采购-2024-12-18 15:25:35
+2024-12-18 15:41:26.000 | INFO     | __main__:detail_get:164 - [采集成功]医疗设备采购项目-2024-12-18 15:25:35
+2024-12-18 15:41:26.795 | INFO     | __main__:detail_get:164 - [采集成功]仁化县丹霞中心小学引进物业公司管理服务-2024-12-18 15:25:12
+2024-12-18 15:41:27.650 | INFO     | __main__:detail_get:164 - [采集成功]2024年秋季印刷品-2024-12-18 15:25:10
+2024-12-18 15:41:28.233 | INFO     | __main__:detail_get:164 - [采集成功]化州市文楼镇23条典型村农村生活污水治理及设施整改提升项目-2024-12-18 15:25:10
+2024-12-18 15:41:28.818 | INFO     | __main__:detail_get:164 - [采集成功]采购一批办公家具-2024-12-18 15:24:50
+2024-12-18 15:41:29.554 | INFO     | __main__:detail_get:164 - [采集成功]采购复印纸-2024-12-18 15:24:37
+2024-12-18 15:41:30.105 | INFO     | __main__:detail_get:164 - [采集成功]采购一台液晶显示器（财务室）-2024-12-18 15:23:59
+2024-12-18 15:41:30.695 | INFO     | __main__:detail_get:164 - [采集成功]物业管理服务-2024-12-18 15:23:34
+2024-12-18 15:41:31.291 | INFO     | __main__:detail_get:164 - [采集成功]郑柏凌 - CGXM20241216012-006 - 触控一体机-2024-12-18 15:23:33
+2024-12-18 15:41:31.880 | INFO     | __main__:detail_get:164 - [采集成功]A4复印纸采购-2024-12-18 15:23:30
+2024-12-18 15:41:32.405 | INFO     | __main__:detail_get:164 - [采集成功]采购台式计算机一批-2024-12-18 15:23:15
+2024-12-18 15:41:33.052 | INFO     | __main__:detail_get:164 - [采集成功]省女子所关于采购无线话筒的计划-2024-12-18 15:23:11
+2024-12-18 15:41:33.628 | INFO     | __main__:detail_get:164 - [采集成功]陆河县市场物业管理总站复印纸采购项目-2024-12-18 15:23:10
+2024-12-18 15:41:34.179 | INFO     | __main__:detail_get:164 - [采集成功]防范和处置非法集资宣传物品-2024-12-18 15:23:05
+2024-12-18 15:41:34.713 | INFO     | __main__:detail_get:164 - [采集成功]佛山市三水区人民医院第四批复印纸采购-2024-12-18 15:22:19
+2024-12-18 15:41:35.276 | INFO     | __main__:detail_get:164 - [采集成功]2025年物业管理服务-2024-12-18 15:22:18
+2024-12-18 15:41:35.858 | INFO     | __main__:detail_get:164 - [采集成功]学校物业管理服务-2024-12-18 15:22:18
+2024-12-18 15:41:36.470 | INFO     | __main__:detail_get:164 - [采集成功]复印纸采购项目-2024-12-18 15:21:55
+2024-12-18 15:41:37.038 | INFO     | __main__:detail_get:164 - [采集成功]退役军人服务站多功能一体机-2024-12-18 15:21:30
+2024-12-18 15:41:37.671 | INFO     | __main__:detail_get:164 - [采集成功]广州市殡葬服务中心殡仪车GPS续费项目-2024-12-18 15:21:04
+2024-12-18 15:41:38.302 | INFO     | __main__:detail_get:164 - [采集成功]2025年度花都点车辆定点加油-2024-12-18 15:21:00
+2024-12-18 15:41:38.908 | INFO     | __main__:detail_get:164 - [采集成功]汕头市潮南区峡山泗联初级中学物业管理采购计划-2024-12-18 15:20:48
+2024-12-18 15:41:39.479 | INFO     | __main__:detail_get:164 - [采集成功]汕头市龙湖区人民法院司法辅助综合事务性工作社会会化服务项目-2024-12-18 15:20:47
+2024-12-18 15:41:40.055 | INFO     | __main__:detail_get:164 - [采集成功]复印纸采购-2024-12-18 15:20:20
+2024-12-18 15:41:40.658 | INFO     | __main__:detail_get:164 - [采集成功]印刷消防宣传资料-2024-12-18 15:20:05
+2024-12-18 15:41:41.227 | INFO     | __main__:detail_get:164 - [采集成功]人大办台式计算机-2024-12-18 15:19:40
+2024-12-18 15:41:41.801 | INFO     | __main__:detail_get:164 - [采集成功]印刷白花镇第十五届人民代表大会第七次会议文件-2024-12-18 15:19:35
+2024-12-18 15:41:42.382 | INFO     | __main__:detail_get:164 - [采集成功]文件、资料及宣传物料印刷服务-2024-12-18 15:19:25
+2024-12-18 15:41:43.080 | INFO     | __main__:detail_get:164 - [采集成功]印刷卫生许可证500张-2024-12-18 15:19:15
+2024-12-18 15:41:43.672 | INFO     | __main__:detail_get:164 - [采集成功]中心办公室付2025年度保安及保洁服务费用-2024-12-18 15:19:00
+2024-12-18 15:41:44.250 | INFO     | __main__:detail_get:164 - [采集成功]印刷文件纸（中共惠东县白花镇委员会）-2024-12-18 15:19:00
+2024-12-18 15:41:45.059 | INFO     | __main__:detail_get:164 - [采集成功]7B2房厕所洗手台修缮等15项零星修缮工程-2024-12-18 15:18:55
+2024-12-18 15:41:45.905 | INFO     | __main__:detail_get:164 - [采集成功]安全生产、禁毒宣传品印刷-2024-12-18 15:18:50
+2024-12-18 15:41:50.088 | INFO     | __main__:detail_get:164 - [采集成功]广州市商务局2024年广州市消费品以旧换新公共服务平台政务信息化项目主体内容采购-2024-12-18 15:18:47
+2024-12-18 15:41:50.702 | INFO     | __main__:detail_get:164 - [采集成功]采购办公家具一批-2024-12-18 15:18:46
+2024-12-18 15:41:51.300 | INFO     | __main__:detail_get:164 - [采集成功]南头人社分局行政法律服务诉讼采购（实毅）-2024-12-18 15:18:45
+2024-12-18 15:41:51.893 | INFO     | __main__:detail_get:164 - [采集成功]印刷项目变更说明、工程项目变更说明、责令整改通知书等文件-2024-12-18 15:18:30
+2024-12-18 15:41:52.462 | INFO     | __main__:detail_get:164 - [采集成功]广州市花都区智慧社区建设项目编制可行性研究报告服务项目-2024-12-18 15:18:30
+2024-12-18 15:41:53.929 | INFO     | __main__:detail_get:164 - [采集成功]2024年活动影集印刷项目-2024-12-18 15:18:20
+2024-12-18 15:41:54.469 | INFO     | __main__:detail_get:164 - [采集成功]2024年12月印刷服务-2024-12-18 15:18:15
+2024-12-18 15:41:55.083 | INFO     | __main__:detail_get:164 - [采集成功]20241218（2301）zzb-新造镇人民政府采购复印纸一批-2024-12-18 15:17:56
+2024-12-18 15:41:55.670 | INFO     | __main__:detail_get:164 - [采集成功]珠海市金湾区三灶镇中心村书院文化展示体验空间项目-第三批家具采购-2024-12-18 15:17:55
+2024-12-18 15:41:56.249 | INFO     | __main__:detail_get:164 - [采集成功]物业管理服务-2024-12-18 15:17:55
+2024-12-18 15:41:57.004 | INFO     | __main__:detail_get:164 - [采集成功]2022年活动影集印刷项目-2024-12-18 15:17:55
+2024-12-18 15:41:57.619 | INFO     | __main__:detail_get:164 - [采集成功]关于办公家具（第一批）的采购计划-2024-12-18 15:17:45
+2024-12-18 15:41:58.195 | INFO     | __main__:detail_get:164 - [采集成功]汕头职业技术学院音乐综合实训室建设项目（装修工程部分）-2024-12-18 15:17:45
+2024-12-18 15:41:58.762 | INFO     | __main__:detail_get:164 - [采集成功]乐购东莞 欢乐长安审计-2024-12-18 15:16:10
+2024-12-18 15:41:58.765 | DEBUG    | __main__:start:233 - ********** 详情页采集结束 **********

lzz_theme/gdszfcgw/start.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+ps -ef |grep "gdszfcgw_daliy_crawl.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+ps -ef |grep "gdszfcgw_details_spider.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 gdszfcgw_daliy_crawl.py > log/gdszfcgw_daliy_crawl.out 2>&1 &
+nohup python3 gdszfcgw_details_spider.py > log/gdszfcgw_details_spider.out 2>&1 &

lzz_theme/hnszfcgdzmc/dt_start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ps -ef |grep "zxjj_details.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 zxjj_details.py > log/zxjj_details.out 2>&1 &
+

lzz_theme/hnszfcgdzmc/hn_collector.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-06-20
+---------
+@summary: 启动器
+---------
+@author: Lzz
+"""
+import glob
+import importlib
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor, wait
+from loguru import logger
+from spider import Crawl_Hndzmc
+
+
+# 获取指定路径下的所有.py文件
+file_list = glob.glob('./*.py')
+
+menus_list = []
+# 遍历并导入 52 个文件  共计 808 页
+# files = os.listdir(os.getcwd())
+# file_list = [file for file in files if re.findall('[\u4e00-\u9fa5]', file)]
+
+for file in file_list:
+    module_name = file[:-3].replace(".\\", "")  # 去除后缀名".py" windows
+    # module_name = file[:-3].replace("./", "")  # 去除后缀名".py"  centos mac
+    if re.findall('^[\u4e00-\u9fa5]', module_name):
+        spider_param = importlib.import_module(module_name)
+        menus_list.append(spider_param.menus)
+
+
+def myfine(menus):
+    Crawl_Hndzmc().start(menus)
+
+
+def main(work=1):
+
+    logger.info(" 启动器 开启 >>> >>> ")
+    start_time = int(time.time())
+    with ThreadPoolExecutor(max_workers=work) as thp:
+
+        future_list = []
+        for menus in menus_list:
+            future = thp.submit(myfine, menus)
+            future_list.append(future)
+
+        wait(future_list)
+
+    end_time = int(time.time())
+    logger.info(f"本轮运行时间：{end_time - start_time} s")
+    logger.info(" <<< <<< 启动器 关闭 ")
+
+
+if __name__ == '__main__':
+    main(work=1)
+

lzz_theme/hnszfcgdzmc/pinyi_jjjg_spider.py

@@ -0,0 +1,249 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-09-12
+---------
+@summary: 湖南省政府采购电子卖场
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+import requests
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from collections import namedtuple
+from utils.RedisDB import RedisFilter
+from utils.tools import *
+import json
+import warnings
+
+
+warnings.filterwarnings('ignore')
+
+# def get_IP():
+#     url = "https://share.proxy.qg.net/get?key=09D1B211&num=1&area=&isp=0&format=txt&seq=&distinct=true"
+#     res = requests.get(url, timeout=10)
+#     proxy = {'http': f'http://{res.text}',
+#              'https': f'http://{res.text}'}
+#     logger.warning(proxy)
+#     return proxy
+
+def get_IP():
+    proxy = "http://CBD8E3AB:E936749E747D@tun-uzqqwl.qg.net:19281"
+    proxies = {
+        "http": proxy,
+        "https": proxy,
+    }
+    return proxies
+
+
+
+class Crawl_Hndzmc:
+
+    def __init__(self):
+        self.py_spider = Mongo_client().py_spider
+        self.zb_list = self.py_spider.theme_list
+        self.RDS = RedisFilter()
+        self.real_cont = 0
+        # self.ip_list = pinyi_proxy()
+        # self.py_proxy = self.ip_list.pop(0)
+        self.py_proxy = get_IP()
+        self.params = {}
+        self.cookies = {}
+
+    def get_acw_sc_v2(self, html):
+        try:
+            arg1 = "".join(re.findall("arg1='(.*?)'", html))
+            if arg1:
+                js_script = '''
+                    function getAcw_sc__v2(obt_arg1) {
+                        String["prototype"]["hexXor"] = function (_0x4e08d8) {
+                            var _0x5a5d3b = '';
+                            for (var _0xe89588 = 0x0; _0xe89588 < this["length"] && _0xe89588 < _0x4e08d8["length"]; _0xe89588 += 2) {
+                                var _0x401af1 = parseInt(this["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x105f59 = parseInt(_0x4e08d8["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x189e2c = (_0x401af1 ^ _0x105f59)["toString"](16);
+                                if (_0x189e2c["length"] == 1) {
+                                    _0x189e2c = '0' + _0x189e2c;
+                                }
+                                _0x5a5d3b += _0x189e2c;
+                            }
+                            return _0x5a5d3b;
+                        };
+                        String["prototype"]["unsbox"] = function () {
+                            var _0x4b082b = [15, 35,29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36];
+                            var _0x4da0dc = [];
+                            var _0x12605e = '';
+                            for (var _0x20a7bf = 0x0; _0x20a7bf < this["length"]; _0x20a7bf++) {
+                                var _0x385ee3 = this[_0x20a7bf];
+                                for (var _0x217721 = 0; _0x217721 < _0x4b082b["length"]; _0x217721++) {
+                                    if (_0x4b082b[_0x217721] == _0x20a7bf + 1) {
+                                        _0x4da0dc[_0x217721] = _0x385ee3;
+                                    }
+                                }
+                            }
+                            _0x12605e = _0x4da0dc["join"]('');
+                            return _0x12605e;
+                        };
+
+                        var _0x5e8b26 = "3000176000856006061501533003690027800375";
+                        // var arg1 = "0A5F01F50F9BC66FB28038F18B99B7B10CFF4667"
+                        var arg1 = obt_arg1
+                        var _0x23a392 = arg1["unsbox"]();
+                        arg2 = _0x23a392["hexXor"](_0x5e8b26);
+                        return arg2
+                    }
+                '''
+                ctx = execjs.compile(js_script)
+                arg2 = ctx.call('getAcw_sc__v2', arg1)
+                return {"acw_sc__v2": arg2}
+            else:
+                return {}
+        except:
+            return {}
+
+    def fetch_list_page(self, page, menu, proxy):
+        logger.debug(f' *** {menu.channel} 开始采集第{page}页 ***')
+
+        session = requests.Session()
+        session.proxies = get_IP()
+        session.verify = False
+
+        headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": "https://hunan.zcygov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://hunan.zcygov.cn/bidding/result",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        data = {
+            "backCategoryName": "",
+            "pageNo": page,
+            "pageSize": 10,
+            "stateList": [],
+            "otherSearch": "",
+            "districtCodeList": [],
+            "sortField": "GMT_MODIFIED",
+            "instanceCode": "HNDZMC",
+            "sortMethod": "DESC"
+        }
+        data = json.dumps(data)
+
+        retry = 0
+        while (retry := retry + 1) < 3:
+            url = "https://hunan.zcygov.cn/front/api/sparta/announcement/listResult"
+            resp = session.post(url, headers=headers, cookies=self.cookies, data=data, timeout=20)
+            self.cookies.update(session.cookies.get_dict())
+
+            arg1_ck = self.get_acw_sc_v2(resp.text)
+
+            if "滑动验证页面" in resp.text:
+                logger.warning("滑动验证页面")
+                return None
+
+            elif arg1_ck:
+                logger.warning("arg1_ck")
+                self.cookies.update(arg1_ck)
+            else:
+                return resp
+
+        return None
+
+    def parser_list_page(self, response, page, menu):
+        results_list = []
+        info_list = response.json().get('result').get('list') or []
+        for info in info_list:
+            title = info.get('title').strip()
+            hid = info.get('requisitionId')
+            tp = info.get('type')
+            href = f'https://hunan.zcygov.cn/bidding/detail?requisitionId={hid}&type={tp}'
+            create_time = timestamp_to_date(int(str(info.get("pubTimestamp"))[:-3]))
+
+            area = "湖南"  # 省份
+            city = info.get('cityName','')
+
+            dedup = [href]
+            if not self.RDS.data_filter(dedup):
+                item = {
+                    "site": "湖南省政府采购电子卖场",
+                    "channel": menu.channel,
+                    "spidercode": menu.spidercode,
+                    "area": area,
+                    "city": city,
+                    "district": "",
+                    "href": href,
+                    "title": title,
+                    "publishtime": create_time,
+                    "parse_url": href,
+                    "parser_name": "ztpc_hnszfcgdzmc",
+                    "is_mixed": False,
+                    "is_theme": True,
+                    "retry": 0,
+                    "comeintime": int2long(int(time.time())),
+                    "is_crawl": False,
+                    "failed": False,
+                    "iscompete": True,
+                    "sendflag": "false",
+                    "T": "bidding",
+                    "infoformat": 1,
+                    "type": "",
+                    "publishdept": "",
+                    "_d": "comeintime",
+                }
+
+                self.zb_list.insert_one(item)
+                self.RDS.data_save_redis(dedup)
+                results_list.append(item)
+
+        logger.info(f' *** {menu.channel}_第{page}页 采集完毕 - 共{len(info_list)}条 - 入库{len(results_list)}条 ***')
+
+        return results_list
+
+    def crawl_list_spider(self, page, menu):
+        retry = 0
+        while (retry := retry + 1) < 10:
+            try:
+                # if len(self.ip_list) < 10:
+                #     self.ip_list = pinyi_proxy()
+                logger.debug(f"{menu.channel}_第{page}页 start")
+                response = self.fetch_list_page(page=page, menu=menu, proxy=self.py_proxy)
+                if response is not None and response.status_code == 200:
+                    informations = self.parser_list_page(response=response, page=page, menu=menu)
+                    crawl_num = len(informations)
+                    self.real_cont += crawl_num
+                    logger.debug(f"{menu.channel}_第 {page} 页 end, 当前已采集 {self.real_cont} 条数据")
+                    time.sleep(random.randint(5,10))
+                    return
+                else:
+                    time.sleep(2)
+                    # self.py_proxy = self.ip_list.pop(0)
+                    self.py_proxy = get_IP()
+            except Exception as e:
+                logger.error(e)
+                time.sleep(2)
+                # self.py_proxy = self.ip_list.pop(0)
+                self.py_proxy = get_IP()
+
+    def start(self, menus):
+        logger.debug("采集开始 》》》 ")
+        for menu in menus:
+            page = menu.crawl_page
+            for page in range(1, page + 1):
+                self.crawl_list_spider(page, menu)
+        logger.debug(" 《《《 采集结束 ")
+
+if __name__ == '__main__':
+    Menu = namedtuple('Menu', ['channel', 'spidercode', 'crawl_page'])
+
+    menus = [
+        Menu('湖南省本级-竞价结果', 'hn_hnszfcgdzmc_hnsbj_jjjg', 200),
+    ]
+
+    Crawl_Hndzmc().start(menus)

lzz_theme/hnszfcgdzmc/pinyi_spider.py

@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-09-12
+---------
+@summary: 湖南省政府采购电子卖场
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+import requests
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from collections import namedtuple
+from utils.RedisDB import RedisFilter
+from utils.attachment import AttachmentDownloader
+from utils.tools import *
+from parsel import Selector
+import json
+import warnings
+
+
+
+warnings.filterwarnings('ignore')
+
+
+
+class Crawl_Hndzmc:
+
+    def __init__(self):
+        self.py_spider = Mongo_client().py_spider
+        self.zb_details = self.py_spider.data_bak
+        self.RDS = RedisFilter()
+        self.real_cont = 0
+        self.proxy = get_proxy()
+        self.params = {}
+        self.cookies = {}
+        self.is_stop = False
+
+    def get_acw_sc_v2(self, html):
+        try:
+            arg1 = "".join(re.findall("arg1='(.*?)'", html))
+            if arg1:
+                js_script = '''
+                    function getAcw_sc__v2(obt_arg1) {
+                        String["prototype"]["hexXor"] = function (_0x4e08d8) {
+                            var _0x5a5d3b = '';
+                            for (var _0xe89588 = 0x0; _0xe89588 < this["length"] && _0xe89588 < _0x4e08d8["length"]; _0xe89588 += 2) {
+                                var _0x401af1 = parseInt(this["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x105f59 = parseInt(_0x4e08d8["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x189e2c = (_0x401af1 ^ _0x105f59)["toString"](16);
+                                if (_0x189e2c["length"] == 1) {
+                                    _0x189e2c = '0' + _0x189e2c;
+                                }
+                                _0x5a5d3b += _0x189e2c;
+                            }
+                            return _0x5a5d3b;
+                        };
+                        String["prototype"]["unsbox"] = function () {
+                            var _0x4b082b = [15, 35,29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36];
+                            var _0x4da0dc = [];
+                            var _0x12605e = '';
+                            for (var _0x20a7bf = 0x0; _0x20a7bf < this["length"]; _0x20a7bf++) {
+                                var _0x385ee3 = this[_0x20a7bf];
+                                for (var _0x217721 = 0; _0x217721 < _0x4b082b["length"]; _0x217721++) {
+                                    if (_0x4b082b[_0x217721] == _0x20a7bf + 1) {
+                                        _0x4da0dc[_0x217721] = _0x385ee3;
+                                    }
+                                }
+                            }
+                            _0x12605e = _0x4da0dc["join"]('');
+                            return _0x12605e;
+                        };
+
+                        var _0x5e8b26 = "3000176000856006061501533003690027800375";
+                        // var arg1 = "0A5F01F50F9BC66FB28038F18B99B7B10CFF4667"
+                        var arg1 = obt_arg1
+                        var _0x23a392 = arg1["unsbox"]();
+                        arg2 = _0x23a392["hexXor"](_0x5e8b26);
+                        return arg2
+                    }
+                '''
+                ctx = execjs.compile(js_script)
+                arg2 = ctx.call('getAcw_sc__v2', arg1)
+                return {"acw_sc__v2": arg2}
+            else:
+                return {}
+        except:
+            return {}
+
+    def fetch_list_page(self, page, menu):
+        logger.debug(f' *** {menu.channel} 开始采集第{page}页 ***')
+
+        session = requests.Session()
+
+        session.proxies = get_QGIP()
+        session.verify = False
+
+        headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": "https://hunan.zcygov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://hunan.zcygov.cn/bidding/result",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        data = {
+            "pageSize": 100,
+            "pageNo": page,
+            "announcementTypes": menu.anTypes,
+            "district": menu.district
+        }
+        data = json.dumps(data)
+
+        retry = 0
+        while (retry := retry + 1) < 3:
+            url = "https://hunan.zcygov.cn/announcement/lobby/queryPage"
+            resp = session.post(url, headers=headers, cookies=self.cookies, data=data, timeout=20)
+            self.cookies.update(session.cookies.get_dict())
+
+            arg1_ck = self.get_acw_sc_v2(resp.text)
+
+            if "滑动验证页面" in resp.text:
+                logger.warning("滑动验证页面")
+                return None
+
+            elif arg1_ck:
+                logger.warning("arg1_ck")
+                self.cookies.update(arg1_ck)
+            else:
+                return resp
+
+        return None
+
+    def parser_list_page(self, response, page, menu):
+        results_list = []
+        info_list = response.json().get('result').get('data') or []
+        for info in info_list:
+            title = info.get('title').strip()
+            encryptId = info.get('encryptId')
+            href = f'https://hunan.zcygov.cn/luban/announcement/detail?encryptId={encryptId}&district={menu.district}'
+            create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(str(info.get("releasedAt"))[:-3])))
+
+            chl = menu.channel
+
+            area = "湖南"  # 省份
+            city = ""
+            if "自治州" in chl:
+                city = chl.split('自治州')[0] + "自治州"
+            elif "市" in chl:
+                city = chl.split('市')[0] + "市"
+
+            html = f'''<span>发布人： {info.get('checkerName')}</span>
+                        <span>来源： {info.get('orgName')}</span>
+                        <span>发布时间： {create_time}</span>{info.get('content')}'''
+
+            contenthtml = html.replace('None', '')
+            detail = cleaner(contenthtml)
+
+            pub_time = handle_publish_time(create_time)
+            publishtime = pub_time[0]
+            l_np_publishtime = pub_time[1]
+
+            dedup = [href]
+            if not self.RDS.data_filter(dedup):
+                attachments = {}
+                root = Selector(info.get('content'))
+                file_list = root.xpath('//a[@href]')
+                if file_list:
+                    ad = AttachmentDownloader()
+                    for ff in file_list:
+                        file_url = ff.xpath('./@href').extract_first()
+                        file_name = ff.xpath('./text()').extract_first("").strip()
+                        file_type = extract_file_type(file_name, file_url)
+                        if not file_type:
+                            continue
+
+                        for _ in range(3):
+                            attachment = ad.fetch_attachment(
+                                file_name=file_name,
+                                file_type=file_type,
+                                download_url=file_url,
+                                proxies=self.proxy
+                            )
+                            if attachment.__contains__("fid"):
+                                attachments[str(len(attachments) + 1)] = attachment
+                                break
+                            self.proxy = get_proxy()
+
+                item = {
+                    "site": "湖南省政府采购电子卖场",
+                    "channel": menu.channel,
+                    "spidercode": menu.spidercode,
+                    "area": area,
+                    "city": city,
+                    "district": "",
+                    "href": href,
+                    "title": title,
+                    "s_title": title,
+                    "publishtime": publishtime,
+                    "l_np_publishtime": l_np_publishtime,
+                    "is_mixed": True,
+                    "comeintime": int2long(int(time.time())),
+                    "contenthtml": contenthtml,
+                    "detail": detail,
+                    "iscompete": True,
+                    "sendflag": "false",
+                    "T": "bidding",
+                    "infoformat": 1,
+                    "type": "",
+                    "publishdept": "",
+                    "_d": "comeintime",
+                }
+
+                if attachments:
+                    item['projectinfo'] = {"attachments": attachments}
+
+                self.zb_details.insert_one(item)
+                self.RDS.data_save_redis(dedup)
+                results_list.append(item)
+
+        logger.info(f' *** {menu.channel}_第{page}页 采集完毕 - 共{len(info_list)}条 - 入库{len(results_list)}条 ***')
+        if len(results_list) < 10 or len(info_list) < 100:
+            self.is_stop = True
+        return results_list
+
+    def crawl_list_spider(self, page, menu):
+        retry = 0
+        while (retry := retry + 1) < 10:
+            try:
+                logger.debug(f"{menu.channel}_第{page}页 start")
+                response = self.fetch_list_page(page=page, menu=menu)
+                if response is not None and response.status_code == 200:
+                    informations = self.parser_list_page(response=response, page=page, menu=menu)
+                    crawl_num = len(informations)
+                    self.real_cont += crawl_num
+                    logger.info(f"{menu.channel}_第 {page} 页 end, 当前已采集 {self.real_cont} 条数据")
+                    time.sleep(random.randint(1,4))
+                    return
+                else:
+                    time.sleep(2)
+
+            except Exception as e:
+                logger.error(e)
+                time.sleep(2)
+
+
+    def start(self, menus):
+        for menu in menus:
+            # page = menu.crawl_page
+            for page in range(1, 100 + 1):
+                self.crawl_list_spider(page, menu)
+                if self.is_stop:
+                    break
+            self.is_stop = False
+
+
+if __name__ == '__main__':
+
+    Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+    announcementTypes = [8020, 8025, 8026, 8013]
+
+    menus = [
+        Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 30),
+    ]
+
+    Crawl_Hndzmc().start(menus)

lzz_theme/hnszfcgdzmc/pinyi_zxjj_spider.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-09-12
+---------
+@summary: 湖南省政府采购电子卖场
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+import requests
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from collections import namedtuple
+from utils.RedisDB import RedisFilter
+from utils.tools import *
+import json
+import warnings
+
+
+warnings.filterwarnings('ignore')
+
+
+class Crawl_Hndzmc:
+
+    def __init__(self):
+        self.py_spider = Mongo_client().py_spider
+        self.zb_list = self.py_spider.theme_list
+        self.RDS = RedisFilter()
+        self.real_cont = 0
+        self.ip_list = pinyi_proxy()
+        self.py_proxy = self.ip_list.pop(0)
+        self.params = {}
+        self.cookies = {}
+
+    def get_acw_sc_v2(self, html):
+        try:
+            arg1 = "".join(re.findall("arg1='(.*?)'", html))
+            if arg1:
+                js_script = '''
+                    function getAcw_sc__v2(obt_arg1) {
+                        String["prototype"]["hexXor"] = function (_0x4e08d8) {
+                            var _0x5a5d3b = '';
+                            for (var _0xe89588 = 0x0; _0xe89588 < this["length"] && _0xe89588 < _0x4e08d8["length"]; _0xe89588 += 2) {
+                                var _0x401af1 = parseInt(this["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x105f59 = parseInt(_0x4e08d8["slice"](_0xe89588, _0xe89588 + 2), 16);
+                                var _0x189e2c = (_0x401af1 ^ _0x105f59)["toString"](16);
+                                if (_0x189e2c["length"] == 1) {
+                                    _0x189e2c = '0' + _0x189e2c;
+                                }
+                                _0x5a5d3b += _0x189e2c;
+                            }
+                            return _0x5a5d3b;
+                        };
+                        String["prototype"]["unsbox"] = function () {
+                            var _0x4b082b = [15, 35,29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36];
+                            var _0x4da0dc = [];
+                            var _0x12605e = '';
+                            for (var _0x20a7bf = 0x0; _0x20a7bf < this["length"]; _0x20a7bf++) {
+                                var _0x385ee3 = this[_0x20a7bf];
+                                for (var _0x217721 = 0; _0x217721 < _0x4b082b["length"]; _0x217721++) {
+                                    if (_0x4b082b[_0x217721] == _0x20a7bf + 1) {
+                                        _0x4da0dc[_0x217721] = _0x385ee3;
+                                    }
+                                }
+                            }
+                            _0x12605e = _0x4da0dc["join"]('');
+                            return _0x12605e;
+                        };
+
+                        var _0x5e8b26 = "3000176000856006061501533003690027800375";
+                        // var arg1 = "0A5F01F50F9BC66FB28038F18B99B7B10CFF4667"
+                        var arg1 = obt_arg1
+                        var _0x23a392 = arg1["unsbox"]();
+                        arg2 = _0x23a392["hexXor"](_0x5e8b26);
+                        return arg2
+                    }
+                '''
+                ctx = execjs.compile(js_script)
+                arg2 = ctx.call('getAcw_sc__v2', arg1)
+                return {"acw_sc__v2": arg2}
+            else:
+                return {}
+        except:
+            return {}
+
+    def fetch_list_page(self, page, menu, proxy):
+        logger.debug(f' *** {menu.channel} 开始采集第{page}页 ***')
+
+        session = requests.Session()
+        session.proxies = proxy
+        session.verify = False
+
+        headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;charset=UTF-8",
+            "Origin": "https://hunan.zcygov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://hunan.zcygov.cn/bidding/newest?tradeModel=BIDDING",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        data = {
+            "backCategoryName": "",
+            "pageNo": page,
+            "pageSize": 100,
+            "stateList": [],
+            "otherSearch": "",
+            "instanceCode": "HNDZMC",
+            "sortField": "ANNOUNCEMENT_PUBLISH_TIME",
+            "sortMethod": "DESC",
+            "districtCodeList": [],
+            "administrativeDistrictCodeList": [],
+            "tradeModel": "BIDDING"
+        }
+        data = json.dumps(data)
+
+        retry = 0
+        while (retry := retry + 1) < 3:
+            url = "https://hunan.zcygov.cn/front/api/sparta/announcement/listNewest"
+            resp = session.post(url, headers=headers, cookies=self.cookies, data=data, timeout=20)
+            self.cookies.update(session.cookies.get_dict())
+
+            arg1_ck = self.get_acw_sc_v2(resp.text)
+
+            if "滑动验证页面" in resp.text:
+                logger.warning("滑动验证页面")
+                return None
+
+            elif arg1_ck:
+                logger.warning("arg1_ck")
+                self.cookies.update(arg1_ck)
+            else:
+                return resp
+
+        return None
+
+    def parser_list_page(self, response, page, menu):
+        results_list = []
+        info_list = response.json().get('result').get('list') or []
+        for info in info_list:
+            title = info.get('title').strip()
+            hid = info.get('requisitionId')
+            tp = info.get('type')
+            href = f'https://hunan.zcygov.cn/bidding/detail?requisitionId={hid}&type={tp}'
+            create_time = timestamp_to_date(int(str(info.get("pubTimestamp"))[:-3]))
+
+            area = "湖南"  # 省份
+            city = info.get('cityName','')
+
+            dedup = [href]
+            if not self.RDS.data_filter(dedup):
+                item = {
+                    "site": "湖南省政府采购电子卖场",
+                    "channel": menu.channel,
+                    "spidercode": menu.spidercode,
+                    "area": area,
+                    "city": city,
+                    "district": "",
+                    "href": href,
+                    "title": title,
+                    "publishtime": create_time,
+                    "parse_url": href,
+                    "parser_name": "ztpc_hnszfcgdzmc",
+                    "is_mixed": False,
+                    "is_theme": True,
+                    "retry": 0,
+                    "comeintime": int2long(int(time.time())),
+                    "is_crawl": False,
+                    "failed": False,
+                    "iscompete": True,
+                    "sendflag": "false",
+                    "T": "bidding",
+                    "infoformat": 1,
+                    "type": "",
+                    "publishdept": "",
+                    "_d": "comeintime",
+                }
+
+                self.zb_list.insert_one(item)
+                self.RDS.data_save_redis(dedup)
+                results_list.append(item)
+
+        logger.info(f' *** {menu.channel}_第{page}页 采集完毕 - 共{len(info_list)}条 - 入库{len(results_list)}条 ***')
+
+        return results_list
+
+    def crawl_list_spider(self, page, menu):
+        retry = 0
+        while (retry := retry + 1) < 10:
+            try:
+                if len(self.ip_list) < 10:
+                    self.ip_list = pinyi_proxy()
+                logger.debug(f"{menu.channel}_第{page}页 start")
+                response = self.fetch_list_page(page=page, menu=menu, proxy=self.py_proxy)
+                if response is not None and response.status_code == 200:
+                    informations = self.parser_list_page(response=response, page=page, menu=menu)
+                    crawl_num = len(informations)
+                    self.real_cont += crawl_num
+                    logger.debug(f"{menu.channel}_第 {page} 页 end, 当前已采集 {self.real_cont} 条数据")
+                    # time.sleep(random.randint(5,10))
+                    return
+                else:
+                    time.sleep(2)
+                    self.py_proxy = self.ip_list.pop(0)
+            except Exception as e:
+                logger.error(e)
+                time.sleep(2)
+                self.py_proxy = self.ip_list.pop(0)
+
+    def start(self, menus):
+        logger.debug("采集开始 》》》 ")
+        for menu in menus:
+            page = menu.crawl_page
+            for page in range(1, page + 1):
+                self.crawl_list_spider(page, menu)
+        logger.debug(" 《《《 采集结束 ")
+
+
+if __name__ == '__main__':
+    Menu = namedtuple('Menu', ['channel', 'spidercode', 'crawl_page'])
+
+
+    menus = [
+        Menu('湖南省本级-最新竞价', 'hn_hnszfcgdzmc_hnsbj_zxjj', 1),
+    ]
+
+    Crawl_Hndzmc().start(menus)

lzz_theme/hnszfcgdzmc/start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+ps -ef |grep "hn_collector.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+ps -ef |grep "zxjj_spider.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+ps -ef |grep "jjjg_spider.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 hn_collector.py > log/hn_collector.out 2>&1 &
+nohup python3 zxjj_spider.py > log/zxjj_spider.out 2>&1 &
+nohup python3 jjjg_spider.py > log/jjjg_spider.out 2>&1 &

lzz_theme/hnszfcgdzmc/处理公告-湖南省.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 处理公告 - 湖南省
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7003, 7004, 1101, 7005, 1102, 7006]
+
+menus = [
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '43', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc(driver_pool=WebCookiePool).start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-娄底市-湘西土家苗族自治州-省本级.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 娄底市 - 湘西土家苗族自治州 - 省本级
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('娄底市娄星区-公告大厅', 'hn_hnszfcgdzmc_ldslxq_ggdt', '431302', announcementTypes, 1),
+    Menu('娄底市双峰县-公告大厅', 'hn_hnszfcgdzmc_ldssfx_ggdt', '431321', announcementTypes, 1),
+    Menu('娄底市新化县-公告大厅', 'hn_hnszfcgdzmc_ldsxhx_ggdt', '431322', announcementTypes, 1),
+    Menu('娄底市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_ldsjjjskfq_ggdt', '431331', announcementTypes, 1),
+    Menu('娄底市冷水江市-公告大厅', 'hn_hnszfcgdzmc_ldslsjs_ggdt', '431381', announcementTypes, 1),
+    Menu('娄底市涟源市-公告大厅', 'hn_hnszfcgdzmc_ldslys_ggdt', '431382', announcementTypes, 1),
+    Menu('娄底市本级-公告大厅', 'hn_hnszfcgdzmc_ldsbj_ggdt', '431399', announcementTypes, 1),
+
+    Menu('湘西土家族苗族自治州吉首市-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzjss_ggdt', '433101', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州泸溪县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlxx_ggdt', '433122', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州凤凰县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzfhx_ggdt', '433123', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州花垣县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzhyx_ggdt', '433124', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州保靖县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzbjx_ggdt', '433125', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州古丈县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzgzx_ggdt', '433126', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州永顺县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzysx_ggdt', '433127', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州龙山县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlsx_ggdt', '433130', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州本级-公告大厅', 'hn_hnszfcgdzmc_xxtjmzzzzbj_ggdt', '433199', announcementTypes, 1),
+
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-常德市-张家界市.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 常德市 - 张家界市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('常德市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_cdsjjjskfq_ggdt', '430701', announcementTypes, 1),
+    Menu('常德市武陵区-公告大厅', 'hn_hnszfcgdzmc_cdswlq_ggdt', '430702', announcementTypes, 1),
+    Menu('常德市鼎城区-公告大厅', 'hn_hnszfcgdzmc_cdsdcq_ggdt', '430703', announcementTypes, 1),
+    Menu('常德市柳叶湖旅游度假区-公告大厅', 'hn_hnszfcgdzmc_cdslyhlydjq_ggdt', '430704', announcementTypes, 1),
+    Menu('常德市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_cdsgxjscykfq_ggdt', '430705', announcementTypes, 1),
+    Menu('常德市西湖管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxhglq_ggdt', '430714', announcementTypes, 1),
+    Menu('常德市安乡县-公告大厅', 'hn_hnszfcgdzmc_cdsaxx_ggdt', '430721', announcementTypes, 1),
+    Menu('常德市汉寿县-公告大厅', 'hn_hnszfcgdzmc_cdshsx_ggdt', '430722', announcementTypes, 1),
+    Menu('常德市澧县-公告大厅', 'hn_hnszfcgdzmc_cdslx_ggdt', '430723', announcementTypes, 1),
+    Menu('常德市临澧县-公告大厅', 'hn_hnszfcgdzmc_cdsllx_ggdt', '430724', announcementTypes, 1),
+    Menu('常德市桃源县-公告大厅', 'hn_hnszfcgdzmc_cdstyx_ggdt', '430725', announcementTypes, 1),
+    Menu('常德市石门县-公告大厅', 'hn_hnszfcgdzmc_cdssmx_ggdt', '430726', announcementTypes, 1),
+    Menu('常德市西洞庭管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxdtglq_ggdt', '430771', announcementTypes, 1),
+    Menu('常德市津市市-公告大厅', 'hn_hnszfcgdzmc_cdsjss_ggdt', '430781', announcementTypes, 1),
+    Menu('常德市常德市本级-公告大厅', 'hn_hnszfcgdzmc_cdsbj_ggdt', '430799', announcementTypes, 1),
+
+    Menu('张家界市永定区-公告大厅', 'hn_hnszfcgdzmc_zjjsydq_ggdt', '430802', announcementTypes, 1),
+    Menu('张家界市武陵源区-公告大厅', 'hn_hnszfcgdzmc_zjjswlyq_ggdt', '430811', announcementTypes, 1),
+    Menu('张家界市慈利县-公告大厅', 'hn_hnszfcgdzmc_zjjsclx_ggdt', '430821', announcementTypes, 1),
+    Menu('张家界市桑植县-公告大厅', 'hn_hnszfcgdzmc_zjjsszx_ggdt', '430822', announcementTypes, 1),
+    Menu('张家界市本级-公告大厅', 'hn_hnszfcgdzmc_zjjsbj_ggdt', '430899', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-永州市-怀化市.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 永州市 - 怀化市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('永州市零陵区-公告大厅', 'hn_hnszfcgdzmc_yzsllq_ggdt', '431102', announcementTypes, 1),
+    Menu('永州市冷水滩区-公告大厅', 'hn_hnszfcgdzmc_yzslstq_ggdt', '431103', announcementTypes, 1),
+    Menu('永州市祁阳县-公告大厅', 'hn_hnszfcgdzmc_yzsqyx_ggdt', '431121', announcementTypes, 1),
+    Menu('永州市东安县-公告大厅', 'hn_hnszfcgdzmc_yzsdax_ggdt', '431122', announcementTypes, 1),
+    Menu('永州市双牌县-公告大厅', 'hn_hnszfcgdzmc_yzsspx_ggdt', '431123', announcementTypes, 1),
+    Menu('永州市道县-公告大厅', 'hn_hnszfcgdzmc_yzsdx_ggdt', '431124', announcementTypes, 1),
+    Menu('永州市江永县-公告大厅', 'hn_hnszfcgdzmc_yzsjyx_ggdt', '431125', announcementTypes, 1),
+    Menu('永州市宁远县-公告大厅', 'hn_hnszfcgdzmc_yzsnyx_ggdt', '431126', announcementTypes, 2),
+    Menu('永州市蓝山县-公告大厅', 'hn_hnszfcgdzmc_yzslsx_ggdt', '431127', announcementTypes, 2),
+    Menu('永州市新田县-公告大厅', 'hn_hnszfcgdzmc_yzsxtx_ggdt', '431128', announcementTypes, 1),
+    Menu('永州市江华瑶族自治县-公告大厅', 'hn_hnszfcgdzmc_yzsjhyzzzx_ggdt', '431129', announcementTypes, 1),
+    Menu('永州市永州市本级-公告大厅', 'hn_hnszfcgdzmc_yzsbj_ggdt', '431199', announcementTypes, 1),
+
+    Menu('怀化市鹤城区-公告大厅', 'hn_hnszfcgdzmc_hhshcq_ggdt', '431202', announcementTypes, 1),
+    Menu('怀化市经济开发区-公告大厅', 'hn_hnszfcgdzmc_hhsjjkfq_ggdt', '431212', announcementTypes, 1),
+    Menu('怀化市洪江管理区-公告大厅', 'hn_hnszfcgdzmc_hhshjglq_ggdt', '431213', announcementTypes, 1),
+    Menu('怀化市中方县-公告大厅', 'hn_hnszfcgdzmc_hhszfx_ggdt', '431221', announcementTypes, 1),
+    Menu('怀化市沅陵县-公告大厅', 'hn_hnszfcgdzmc_hhsylx_ggdt', '431222', announcementTypes, 1),
+    Menu('怀化市辰溪县-公告大厅', 'hn_hnszfcgdzmc_hhscxx_ggdt', '431223', announcementTypes, 1),
+    Menu('怀化市溆浦县-公告大厅', 'hn_hnszfcgdzmc_hhsxpx_ggdt', '431224', announcementTypes, 1),
+    Menu('怀化市会同县-公告大厅', 'hn_hnszfcgdzmc_hhshtx_ggdt', '431225', announcementTypes, 1),
+    Menu('怀化市麻阳苗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsmymzzzx_ggdt', '431226', announcementTypes, 1),
+    Menu('怀化市新晃侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsxhdzzzx_ggdt', '431227', announcementTypes, 1),
+    Menu('怀化市芷江侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhszjdzzzx_ggdt', '431228', announcementTypes, 1),
+    Menu('怀化市靖州苗族侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsjzmzdzzzx_ggdt', '431229', announcementTypes, 1),
+    Menu('怀化市通道侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhstddzzzx_ggdt', '431230', announcementTypes, 1),
+    Menu('怀化市洪江市-公告大厅', 'hn_hnszfcgdzmc_hhshjs_ggdt', '431281', announcementTypes, 1),
+    Menu('怀化市本级-公告大厅', 'hn_hnszfcgdzmc_hhsbj_ggdt', '431299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-湘潭市-衡阳市.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 湘潭市 - 衡阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('湘潭市雨湖区-公告大厅', 'hn_hnszfcgdzmc_xtsyhq_ggdt', '430302', announcementTypes, 1),
+    Menu('湘潭市岳塘区-公告大厅', 'hn_hnszfcgdzmc_xtsytq_ggdt', '430304', announcementTypes, 1),
+    Menu('湘潭市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_xtsgxjscykfq_ggdt', '430311', announcementTypes, 1),
+    Menu('湘潭经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_xtjjjskfq_ggdt', '430312', announcementTypes, 1),
+    Menu('湘潭市湘潭县-公告大厅', 'hn_hnszfcgdzmc_xtsxtx_ggdt', '430321', announcementTypes, 1),
+    Menu('湘潭市湘乡市-公告大厅', 'hn_hnszfcgdzmc_xtsxxs_ggdt', '430381', announcementTypes, 1),
+    Menu('湘潭市韶山市-公告大厅', 'hn_hnszfcgdzmc_xtssss_ggdt', '430382', announcementTypes, 1),
+    Menu('湘潭市本级-公告大厅', 'hn_hnszfcgdzmc_xtsbj_ggdt', '430399', announcementTypes, 1),
+
+    Menu('衡阳市珠晖区-公告大厅', 'hn_hnszfcgdzmc_hyszhq_ggdt', '430405', announcementTypes, 1),
+    Menu('衡阳市雁峰区-公告大厅', 'hn_hnszfcgdzmc_hysyfq_ggdt', '430406', announcementTypes, 1),
+    Menu('衡阳市石鼓区-公告大厅', 'hn_hnszfcgdzmc_hyssgq_ggdt', '430407', announcementTypes, 1),
+    Menu('衡阳市蒸湘区-公告大厅', 'hn_hnszfcgdzmc_hyszxq_ggdt', '430408', announcementTypes, 1),
+    Menu('衡阳市南岳区-公告大厅', 'hn_hnszfcgdzmc_hysnyq_ggdt', '430412', announcementTypes, 1),
+    Menu('衡阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_hysgxjscykfq_ggdt', '430415', announcementTypes, 1),
+    Menu('衡阳市衡阳县-公告大厅', 'hn_hnszfcgdzmc_hyshyx_ggdt', '430421', announcementTypes, 1),
+    Menu('衡阳市衡南县-公告大厅', 'hn_hnszfcgdzmc_hyshnx_ggdt', '430422', announcementTypes, 1),
+    Menu('衡阳市衡山县-公告大厅', 'hn_hnszfcgdzmc_hyshsx_ggdt', '430423', announcementTypes, 1),
+    Menu('衡阳市衡东县-公告大厅', 'hn_hnszfcgdzmc_hyshdx_ggdt', '430424', announcementTypes, 1),
+    Menu('衡阳市祁东县-公告大厅', 'hn_hnszfcgdzmc_hysqdx_ggdt', '430426', announcementTypes, 1),
+    Menu('衡阳市耒阳市-公告大厅', 'hn_hnszfcgdzmc_hyslys_ggdt', '430481', announcementTypes, 2),
+    Menu('衡阳市常宁市-公告大厅', 'hn_hnszfcgdzmc_hyscns_ggdt', '430482', announcementTypes, 1),
+    Menu('衡阳市本级-公告大厅', 'hn_hnszfcgdzmc_hysbj_ggdt', '430499', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-益阳市-郴州市.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 益阳市 - 郴州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('益阳市资阳区-公告大厅', 'hn_hnszfcgdzmc_yyszyq_ggdt', '430902', announcementTypes, 1),
+    Menu('益阳市赫山区-公告大厅', 'hn_hnszfcgdzmc_yyshsq_ggdt', '430903', announcementTypes, 1),
+    Menu('益阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_yysgxjscykfq_ggdt', '430911', announcementTypes, 1),
+    Menu('益阳市大通湖区-公告大厅', 'hn_hnszfcgdzmc_yysdthq_ggdt', '430912', announcementTypes, 1),
+    Menu('益阳市南县-公告大厅', 'hn_hnszfcgdzmc_yysnx_ggdt', '430921', announcementTypes, 1),
+    Menu('益阳市桃江县-公告大厅', 'hn_hnszfcgdzmc_yystjx_ggdt', '430922', announcementTypes, 1),
+    Menu('益阳市安化县-公告大厅', 'hn_hnszfcgdzmc_yysahx_ggdt', '430923', announcementTypes, 1),
+    Menu('益阳市沅江市-公告大厅', 'hn_hnszfcgdzmc_yysyjs_ggdt', '430981', announcementTypes, 1),
+    Menu('益阳市本级-公告大厅', 'hn_hnszfcgdzmc_yiysbj_ggdt', '430999', announcementTypes, 1),
+
+    Menu('郴州市北湖区-公告大厅', 'hn_hnszfcgdzmc_czsbhq_ggdt', '431002', announcementTypes, 1),
+    Menu('郴州市苏仙区-公告大厅', 'hn_hnszfcgdzmc_czssxq_ggdt', '431003', announcementTypes, 1),
+    Menu('郴州市桂阳县-公告大厅', 'hn_hnszfcgdzmc_czsgyx_ggdt', '431021', announcementTypes, 1),
+    Menu('郴州市宜章县-公告大厅', 'hn_hnszfcgdzmc_czsyzx_ggdt', '431022', announcementTypes, 1),
+    Menu('郴州市永兴县-公告大厅', 'hn_hnszfcgdzmc_czsyxx_ggdt', '431023', announcementTypes, 1),
+    Menu('郴州市嘉禾县-公告大厅', 'hn_hnszfcgdzmc_czsjhx_ggdt', '431024', announcementTypes, 1),
+    Menu('郴州市临武县-公告大厅', 'hn_hnszfcgdzmc_czslwx_ggdt', '431025', announcementTypes, 1),
+    Menu('郴州市汝城县-公告大厅', 'hn_hnszfcgdzmc_czsrcx_ggdt', '431026', announcementTypes, 1),
+    Menu('郴州市桂东县-公告大厅', 'hn_hnszfcgdzmc_czsgdx_ggdt', '431027', announcementTypes, 1),
+    Menu('郴州市安仁县-公告大厅', 'hn_hnszfcgdzmc_czsarx_ggdt', '431028', announcementTypes, 1),
+    Menu('郴州市资兴市-公告大厅', 'hn_hnszfcgdzmc_czszxs_ggdt', '431081', announcementTypes, 1),
+    Menu('郴州市本级-公告大厅', 'hn_hnszfcgdzmc_czsbj_ggdt', '431099', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-邵阳市-岳阳市.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 邵阳市 - 岳阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('邵阳市双清区-公告大厅', 'hn_hnszfcgdzmc_syssqq_ggdt', '430502', announcementTypes, 1),
+    Menu('邵阳市大祥区-公告大厅', 'hn_hnszfcgdzmc_sysdxq_ggdt', '430503', announcementTypes, 1),
+    Menu('邵阳市北塔区-公告大厅', 'hn_hnszfcgdzmc_sysbtq_ggdt', '430511', announcementTypes, 1),
+    Menu('邵阳市邵东市-公告大厅', 'hn_hnszfcgdzmc_syssds_ggdt', '430521', announcementTypes, 1),
+    Menu('邵阳市新邵县-公告大厅', 'hn_hnszfcgdzmc_sysxsx_ggdt', '430522', announcementTypes, 1),
+    Menu('邵阳市邵阳县-公告大厅', 'hn_hnszfcgdzmc_syssyx_ggdt', '430523', announcementTypes, 1),
+    Menu('邵阳市隆回县-公告大厅', 'hn_hnszfcgdzmc_syslhx_ggdt', '430524', announcementTypes, 1),
+    Menu('邵阳市洞口县-公告大厅', 'hn_hnszfcgdzmc_sysdkx_ggdt', '430525', announcementTypes, 1),
+    Menu('邵阳市绥宁县-公告大厅', 'hn_hnszfcgdzmc_syssnx_ggdt', '430527', announcementTypes, 1),
+    Menu('邵阳市新宁县-公告大厅', 'hn_hnszfcgdzmc_sysxnx_ggdt', '430528', announcementTypes, 1),
+    Menu('邵阳市城步苗族自治县-公告大厅', 'hn_hnszfcgdzmc_syscbmzzzx_ggdt', '430529', announcementTypes, 1),
+    Menu('邵阳市邵阳经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_syssyjjkfq_ggdt', '430541', announcementTypes, 1),
+    Menu('邵阳市武冈市-公告大厅', 'hn_hnszfcgdzmc_syswgs_ggdt', '430581', announcementTypes, 1),
+    Menu('邵阳市本级-公告大厅', 'hn_hnszfcgdzmc_sysbj_ggdt', '430599', announcementTypes, 1),
+
+    Menu('岳阳市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_yysyyjjjskfq_ggdt', '430601', announcementTypes, 1),
+    Menu('岳阳市岳阳楼区-公告大厅', 'hn_hnszfcgdzmc_yysyylq_ggdt', '430602', announcementTypes, 1),
+    Menu('岳阳市云溪区-公告大厅', 'hn_hnszfcgdzmc_yysyxq_ggdt', '430603', announcementTypes, 1),
+    Menu('岳阳市君山区-公告大厅', 'hn_hnszfcgdzmc_yysjsq_ggdt', '430611', announcementTypes, 1),
+    Menu('岳阳市岳阳县-公告大厅', 'hn_hnszfcgdzmc_yysyyx_ggdt', '430621', announcementTypes, 1),
+    Menu('岳阳市华容县-公告大厅', 'hn_hnszfcgdzmc_yyshrx_ggdt', '430623', announcementTypes, 1),
+    Menu('岳阳市湘阴县-公告大厅', 'hn_hnszfcgdzmc_yysxyx_ggdt', '430624', announcementTypes, 1),
+    Menu('岳阳市平江县-公告大厅', 'hn_hnszfcgdzmc_yyspjx_ggdt', '430626', announcementTypes, 1),
+    Menu('岳阳市南湖新区-公告大厅', 'hn_hnszfcgdzmc_yysnhxq_ggdt', '430632', announcementTypes, 1),
+    Menu('岳阳市城陵矶新港区-公告大厅', 'hn_hnszfcgdzmc_yyscljxgq_ggdt', '430634', announcementTypes, 1),
+    Menu('岳阳市屈原管理区-公告大厅', 'hn_hnszfcgdzmc_yysqyglq_ggdt', '430671', announcementTypes, 1),
+    Menu('岳阳市汨罗市-公告大厅', 'hn_hnszfcgdzmc_yysmls_ggdt', '430681', announcementTypes, 1),
+    Menu('岳阳市临湘市-公告大厅', 'hn_hnszfcgdzmc_yyslxs_ggdt', '430682', announcementTypes, 1),
+    Menu('岳阳市本级-公告大厅', 'hn_hnszfcgdzmc_yysbj_ggdt', '430699', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/异常公告-长沙市-株洲市.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 异常公告 - 长沙市 - 株洲市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8015]
+
+menus = [
+    Menu('长沙市芙蓉区-公告大厅', 'hn_hnszfcgdzmc_cssfrq_ggdt', '430102', announcementTypes, 1),
+    Menu('长沙市天心区-公告大厅', 'hn_hnszfcgdzmc_csstxq_ggdt', '430103', announcementTypes, 1),
+    Menu('长沙市岳麓区-公告大厅', 'hn_hnszfcgdzmc_cssylq_ggdt', '430104', announcementTypes, 1),
+    Menu('长沙市开福区-公告大厅', 'hn_hnszfcgdzmc_csskfq_ggdt', '430105', announcementTypes, 1),
+    Menu('长沙市雨花区-公告大厅', 'hn_hnszfcgdzmc_cssyhq_ggdt', '430111', announcementTypes, 1),
+    Menu('长沙市望城区-公告大厅', 'hn_hnszfcgdzmc_csswcq_ggdt', '430112', announcementTypes, 1),
+    Menu('长沙市长沙县-公告大厅', 'hn_hnszfcgdzmc_csscsx_ggdt', '430121', announcementTypes, 1),
+    Menu('长沙市湖南湘江新区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csshnxjxqglwyh_ggdt', '430132', announcementTypes, 1),
+    Menu('长沙市浏阳市-公告大厅', 'hn_hnszfcgdzmc_csslys_ggdt', '430181', announcementTypes, 1),
+    Menu('长沙市宁乡市-公告大厅', 'hn_hnszfcgdzmc_cssnxs_ggdt', '430182', announcementTypes, 1),
+    Menu('长沙市长沙高新技术产业开发区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csscsgxqglwyh_ggdt', '430131', announcementTypes, 1),
+    Menu('长沙市本级-公告大厅', 'hn_hnszfcgdzmc_cssbj_ggdt', '430199', announcementTypes, 1),
+
+    Menu('株洲市荷塘区-公告大厅', 'hn_hnszfcgdzmc_zzshtq_ggdt', '430202', announcementTypes, 1),
+    Menu('株洲市芦淞区-公告大厅', 'hn_hnszfcgdzmc_zzslsq_ggdt', '430203', announcementTypes, 1),
+    Menu('株洲市石峰区-公告大厅', 'hn_hnszfcgdzmc_zzssfq_ggdt', '430204', announcementTypes, 1),
+    Menu('株洲市天元区-公告大厅', 'hn_hnszfcgdzmc_zzstyq_ggdt', '430211', announcementTypes, 1),
+    Menu('株洲市渌口区-公告大厅', 'hn_hnszfcgdzmc_zzslkq_ggdt', '430212', announcementTypes, 1),
+    Menu('株洲市攸县-公告大厅', 'hn_hnszfcgdzmc_zzsyx_ggdt', '430223', announcementTypes, 1),
+    Menu('株洲市茶陵县-公告大厅', 'hn_hnszfcgdzmc_zzsclx_ggdt', '430224', announcementTypes, 1),
+    Menu('株洲市炎陵县-公告大厅', 'hn_hnszfcgdzmc_zzsylx_ggdt', '430225', announcementTypes, 1),
+    Menu('株洲市株洲云龙示范区-公告大厅', 'hn_hnszfcgdzmc_zzszzylsfq_ggdt', '430232', announcementTypes, 1),
+    Menu('株洲市醴陵市-公告大厅', 'hn_hnszfcgdzmc_zzslls_ggdt', '430281', announcementTypes, 1),
+    Menu('株洲市本级-公告大厅', 'hn_hnszfcgdzmc_zzsbj_ggdt', '430299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-娄底市.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 娄底市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('娄底市娄星区-公告大厅', 'hn_hnszfcgdzmc_ldslxq_ggdt', '431302', announcementTypes, 1),
+    Menu('娄底市双峰县-公告大厅', 'hn_hnszfcgdzmc_ldssfx_ggdt', '431321', announcementTypes, 1),
+    Menu('娄底市新化县-公告大厅', 'hn_hnszfcgdzmc_ldsxhx_ggdt', '431322', announcementTypes, 1),
+    Menu('娄底市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_ldsjjjskfq_ggdt', '431331', announcementTypes, 1),
+    Menu('娄底市冷水江市-公告大厅', 'hn_hnszfcgdzmc_ldslsjs_ggdt', '431381', announcementTypes, 1),
+    Menu('娄底市涟源市-公告大厅', 'hn_hnszfcgdzmc_ldslys_ggdt', '431382', announcementTypes, 1),
+    Menu('娄底市本级-公告大厅', 'hn_hnszfcgdzmc_ldsbj_ggdt', '431399', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-岳阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 岳阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('岳阳市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_yysyyjjjskfq_ggdt', '430601', announcementTypes, 1),
+    Menu('岳阳市岳阳楼区-公告大厅', 'hn_hnszfcgdzmc_yysyylq_ggdt', '430602', announcementTypes, 1),
+    Menu('岳阳市云溪区-公告大厅', 'hn_hnszfcgdzmc_yysyxq_ggdt', '430603', announcementTypes, 1),
+    Menu('岳阳市君山区-公告大厅', 'hn_hnszfcgdzmc_yysjsq_ggdt', '430611', announcementTypes, 1),
+    Menu('岳阳市岳阳县-公告大厅', 'hn_hnszfcgdzmc_yysyyx_ggdt', '430621', announcementTypes, 1),
+    Menu('岳阳市华容县-公告大厅', 'hn_hnszfcgdzmc_yyshrx_ggdt', '430623', announcementTypes, 1),
+    Menu('岳阳市湘阴县-公告大厅', 'hn_hnszfcgdzmc_yysxyx_ggdt', '430624', announcementTypes, 1),
+    Menu('岳阳市平江县-公告大厅', 'hn_hnszfcgdzmc_yyspjx_ggdt', '430626', announcementTypes, 1),
+    Menu('岳阳市南湖新区-公告大厅', 'hn_hnszfcgdzmc_yysnhxq_ggdt', '430632', announcementTypes, 1),
+    Menu('岳阳市城陵矶新港区-公告大厅', 'hn_hnszfcgdzmc_yyscljxgq_ggdt', '430634', announcementTypes, 1),
+    Menu('岳阳市屈原管理区-公告大厅', 'hn_hnszfcgdzmc_yysqyglq_ggdt', '430671', announcementTypes, 1),
+    Menu('岳阳市汨罗市-公告大厅', 'hn_hnszfcgdzmc_yysmls_ggdt', '430681', announcementTypes, 1),
+    Menu('岳阳市临湘市-公告大厅', 'hn_hnszfcgdzmc_yyslxs_ggdt', '430682', announcementTypes, 1),
+    Menu('岳阳市本级-公告大厅', 'hn_hnszfcgdzmc_yysbj_ggdt', '430699', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-常德市.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 常德市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('常德市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_cdsjjjskfq_ggdt', '430701', announcementTypes, 1),
+    Menu('常德市武陵区-公告大厅', 'hn_hnszfcgdzmc_cdswlq_ggdt', '430702', announcementTypes, 1),
+    Menu('常德市鼎城区-公告大厅', 'hn_hnszfcgdzmc_cdsdcq_ggdt', '430703', announcementTypes, 1),
+    Menu('常德市柳叶湖旅游度假区-公告大厅', 'hn_hnszfcgdzmc_cdslyhlydjq_ggdt', '430704', announcementTypes, 1),
+    Menu('常德市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_cdsgxjscykfq_ggdt', '430705', announcementTypes, 1),
+    Menu('常德市西湖管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxhglq_ggdt', '430714', announcementTypes, 1),
+    Menu('常德市安乡县-公告大厅', 'hn_hnszfcgdzmc_cdsaxx_ggdt', '430721', announcementTypes, 1),
+    Menu('常德市汉寿县-公告大厅', 'hn_hnszfcgdzmc_cdshsx_ggdt', '430722', announcementTypes, 1),
+    Menu('常德市澧县-公告大厅', 'hn_hnszfcgdzmc_cdslx_ggdt', '430723', announcementTypes, 1),
+    Menu('常德市临澧县-公告大厅', 'hn_hnszfcgdzmc_cdsllx_ggdt', '430724', announcementTypes, 1),
+    Menu('常德市桃源县-公告大厅', 'hn_hnszfcgdzmc_cdstyx_ggdt', '430725', announcementTypes, 1),
+    Menu('常德市石门县-公告大厅', 'hn_hnszfcgdzmc_cdssmx_ggdt', '430726', announcementTypes, 1),
+    Menu('常德市西洞庭管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxdtglq_ggdt', '430771', announcementTypes, 1),
+    Menu('常德市津市市-公告大厅', 'hn_hnszfcgdzmc_cdsjss_ggdt', '430781', announcementTypes, 1),
+    Menu('常德市常德市本级-公告大厅', 'hn_hnszfcgdzmc_cdsbj_ggdt', '430799', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-张家界市.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 张家界市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('张家界市永定区-公告大厅', 'hn_hnszfcgdzmc_zjjsydq_ggdt', '430802', announcementTypes, 1),
+    Menu('张家界市武陵源区-公告大厅', 'hn_hnszfcgdzmc_zjjswlyq_ggdt', '430811', announcementTypes, 1),
+    Menu('张家界市慈利县-公告大厅', 'hn_hnszfcgdzmc_zjjsclx_ggdt', '430821', announcementTypes, 1),
+    Menu('张家界市桑植县-公告大厅', 'hn_hnszfcgdzmc_zjjsszx_ggdt', '430822', announcementTypes, 1),
+    Menu('张家界市本级-公告大厅', 'hn_hnszfcgdzmc_zjjsbj_ggdt', '430899', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-怀化市.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 怀化市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('怀化市鹤城区-公告大厅', 'hn_hnszfcgdzmc_hhshcq_ggdt', '431202', announcementTypes, 1),
+    Menu('怀化市经济开发区-公告大厅', 'hn_hnszfcgdzmc_hhsjjkfq_ggdt', '431212', announcementTypes, 1),
+    Menu('怀化市洪江管理区-公告大厅', 'hn_hnszfcgdzmc_hhshjglq_ggdt', '431213', announcementTypes, 1),
+    Menu('怀化市中方县-公告大厅', 'hn_hnszfcgdzmc_hhszfx_ggdt', '431221', announcementTypes, 1),
+    Menu('怀化市沅陵县-公告大厅', 'hn_hnszfcgdzmc_hhsylx_ggdt', '431222', announcementTypes, 1),
+    Menu('怀化市辰溪县-公告大厅', 'hn_hnszfcgdzmc_hhscxx_ggdt', '431223', announcementTypes, 1),
+    Menu('怀化市溆浦县-公告大厅', 'hn_hnszfcgdzmc_hhsxpx_ggdt', '431224', announcementTypes, 1),
+    Menu('怀化市会同县-公告大厅', 'hn_hnszfcgdzmc_hhshtx_ggdt', '431225', announcementTypes, 1),
+    Menu('怀化市麻阳苗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsmymzzzx_ggdt', '431226', announcementTypes, 1),
+    Menu('怀化市新晃侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsxhdzzzx_ggdt', '431227', announcementTypes, 1),
+    Menu('怀化市芷江侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhszjdzzzx_ggdt', '431228', announcementTypes, 1),
+    Menu('怀化市靖州苗族侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsjzmzdzzzx_ggdt', '431229', announcementTypes, 1),
+    Menu('怀化市通道侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhstddzzzx_ggdt', '431230', announcementTypes, 1),
+    Menu('怀化市洪江市-公告大厅', 'hn_hnszfcgdzmc_hhshjs_ggdt', '431281', announcementTypes, 1),
+    Menu('怀化市本级-公告大厅', 'hn_hnszfcgdzmc_hhsbj_ggdt', '431299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-株洲市.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 株洲市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('株洲市荷塘区-公告大厅', 'hn_hnszfcgdzmc_zzshtq_ggdt', '430202', announcementTypes, 1),
+    Menu('株洲市芦淞区-公告大厅', 'hn_hnszfcgdzmc_zzslsq_ggdt', '430203', announcementTypes, 1),
+    Menu('株洲市石峰区-公告大厅', 'hn_hnszfcgdzmc_zzssfq_ggdt', '430204', announcementTypes, 1),
+    Menu('株洲市天元区-公告大厅', 'hn_hnszfcgdzmc_zzstyq_ggdt', '430211', announcementTypes, 1),
+    Menu('株洲市渌口区-公告大厅', 'hn_hnszfcgdzmc_zzslkq_ggdt', '430212', announcementTypes, 1),
+    Menu('株洲市攸县-公告大厅', 'hn_hnszfcgdzmc_zzsyx_ggdt', '430223', announcementTypes, 1),
+    Menu('株洲市茶陵县-公告大厅', 'hn_hnszfcgdzmc_zzsclx_ggdt', '430224', announcementTypes, 1),
+    Menu('株洲市炎陵县-公告大厅', 'hn_hnszfcgdzmc_zzsylx_ggdt', '430225', announcementTypes, 1),
+    Menu('株洲市株洲云龙示范区-公告大厅', 'hn_hnszfcgdzmc_zzszzylsfq_ggdt', '430232', announcementTypes, 1),
+    Menu('株洲市醴陵市-公告大厅', 'hn_hnszfcgdzmc_zzslls_ggdt', '430281', announcementTypes, 1),
+    Menu('株洲市本级-公告大厅', 'hn_hnszfcgdzmc_zzsbj_ggdt', '430299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-永州市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 永州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('永州市零陵区-公告大厅', 'hn_hnszfcgdzmc_yzsllq_ggdt', '431102', announcementTypes, 1),
+    Menu('永州市冷水滩区-公告大厅', 'hn_hnszfcgdzmc_yzslstq_ggdt', '431103', announcementTypes, 1),
+    Menu('永州市祁阳县-公告大厅', 'hn_hnszfcgdzmc_yzsqyx_ggdt', '431121', announcementTypes, 1),
+    Menu('永州市东安县-公告大厅', 'hn_hnszfcgdzmc_yzsdax_ggdt', '431122', announcementTypes, 1),
+    Menu('永州市双牌县-公告大厅', 'hn_hnszfcgdzmc_yzsspx_ggdt', '431123', announcementTypes, 1),
+    Menu('永州市道县-公告大厅', 'hn_hnszfcgdzmc_yzsdx_ggdt', '431124', announcementTypes, 1),
+    Menu('永州市江永县-公告大厅', 'hn_hnszfcgdzmc_yzsjyx_ggdt', '431125', announcementTypes, 1),
+    Menu('永州市宁远县-公告大厅', 'hn_hnszfcgdzmc_yzsnyx_ggdt', '431126', announcementTypes, 1),
+    Menu('永州市蓝山县-公告大厅', 'hn_hnszfcgdzmc_yzslsx_ggdt', '431127', announcementTypes, 1),
+    Menu('永州市新田县-公告大厅', 'hn_hnszfcgdzmc_yzsxtx_ggdt', '431128', announcementTypes, 1),
+    Menu('永州市江华瑶族自治县-公告大厅', 'hn_hnszfcgdzmc_yzsjhyzzzx_ggdt', '431129', announcementTypes, 1),
+    Menu('永州市永州市本级-公告大厅', 'hn_hnszfcgdzmc_yzsbj_ggdt', '431199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-湘潭市.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 湘潭市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('湘潭市雨湖区-公告大厅', 'hn_hnszfcgdzmc_xtsyhq_ggdt', '430302', announcementTypes, 1),
+    Menu('湘潭市岳塘区-公告大厅', 'hn_hnszfcgdzmc_xtsytq_ggdt', '430304', announcementTypes, 1),
+    Menu('湘潭市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_xtsgxjscykfq_ggdt', '430311', announcementTypes, 1),
+    Menu('湘潭经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_xtjjjskfq_ggdt', '430312', announcementTypes, 1),
+    Menu('湘潭市湘潭县-公告大厅', 'hn_hnszfcgdzmc_xtsxtx_ggdt', '430321', announcementTypes, 1),
+    Menu('湘潭市湘乡市-公告大厅', 'hn_hnszfcgdzmc_xtsxxs_ggdt', '430381', announcementTypes, 1),
+    Menu('湘潭市韶山市-公告大厅', 'hn_hnszfcgdzmc_xtssss_ggdt', '430382', announcementTypes, 1),
+    Menu('湘潭市本级-公告大厅', 'hn_hnszfcgdzmc_xtsbj_ggdt', '430399', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-湘西土家苗族自治州.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 湘西土家族苗族自治州
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('湘西土家族苗族自治州吉首市-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzjss_ggdt', '433101', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州泸溪县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlxx_ggdt', '433122', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州凤凰县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzfhx_ggdt', '433123', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州花垣县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzhyx_ggdt', '433124', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州保靖县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzbjx_ggdt', '433125', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州古丈县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzgzx_ggdt', '433126', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州永顺县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzysx_ggdt', '433127', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州龙山县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlsx_ggdt', '433130', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州本级-公告大厅', 'hn_hnszfcgdzmc_xxtjmzzzzbj_ggdt', '433199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-益阳市.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 益阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('益阳市资阳区-公告大厅', 'hn_hnszfcgdzmc_yyszyq_ggdt', '430902', announcementTypes, 1),
+    Menu('益阳市赫山区-公告大厅', 'hn_hnszfcgdzmc_yyshsq_ggdt', '430903', announcementTypes, 1),
+    Menu('益阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_yysgxjscykfq_ggdt', '430911', announcementTypes, 1),
+    Menu('益阳市大通湖区-公告大厅', 'hn_hnszfcgdzmc_yysdthq_ggdt', '430912', announcementTypes, 1),
+    Menu('益阳市南县-公告大厅', 'hn_hnszfcgdzmc_yysnx_ggdt', '430921', announcementTypes, 1),
+    Menu('益阳市桃江县-公告大厅', 'hn_hnszfcgdzmc_yystjx_ggdt', '430922', announcementTypes, 1),
+    Menu('益阳市安化县-公告大厅', 'hn_hnszfcgdzmc_yysahx_ggdt', '430923', announcementTypes, 1),
+    Menu('益阳市沅江市-公告大厅', 'hn_hnszfcgdzmc_yysyjs_ggdt', '430981', announcementTypes, 1),
+    Menu('益阳市本级-公告大厅', 'hn_hnszfcgdzmc_yiysbj_ggdt', '430999', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-省本级.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 省本级
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 30),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-衡阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 衡阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('衡阳市珠晖区-公告大厅', 'hn_hnszfcgdzmc_hyszhq_ggdt', '430405', announcementTypes, 1),
+    Menu('衡阳市雁峰区-公告大厅', 'hn_hnszfcgdzmc_hysyfq_ggdt', '430406', announcementTypes, 1),
+    Menu('衡阳市石鼓区-公告大厅', 'hn_hnszfcgdzmc_hyssgq_ggdt', '430407', announcementTypes, 1),
+    Menu('衡阳市蒸湘区-公告大厅', 'hn_hnszfcgdzmc_hyszxq_ggdt', '430408', announcementTypes, 1),
+    Menu('衡阳市南岳区-公告大厅', 'hn_hnszfcgdzmc_hysnyq_ggdt', '430412', announcementTypes, 1),
+    Menu('衡阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_hysgxjscykfq_ggdt', '430415', announcementTypes, 1),
+    Menu('衡阳市衡阳县-公告大厅', 'hn_hnszfcgdzmc_hyshyx_ggdt', '430421', announcementTypes, 1),
+    Menu('衡阳市衡南县-公告大厅', 'hn_hnszfcgdzmc_hyshnx_ggdt', '430422', announcementTypes, 1),
+    Menu('衡阳市衡山县-公告大厅', 'hn_hnszfcgdzmc_hyshsx_ggdt', '430423', announcementTypes, 1),
+    Menu('衡阳市衡东县-公告大厅', 'hn_hnszfcgdzmc_hyshdx_ggdt', '430424', announcementTypes, 1),
+    Menu('衡阳市祁东县-公告大厅', 'hn_hnszfcgdzmc_hysqdx_ggdt', '430426', announcementTypes, 1),
+    Menu('衡阳市耒阳市-公告大厅', 'hn_hnszfcgdzmc_hyslys_ggdt', '430481', announcementTypes, 1),
+    Menu('衡阳市常宁市-公告大厅', 'hn_hnszfcgdzmc_hyscns_ggdt', '430482', announcementTypes, 1),
+    Menu('衡阳市本级-公告大厅', 'hn_hnszfcgdzmc_hysbj_ggdt', '430499', announcementTypes, 2),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-邵阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 邵阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('邵阳市双清区-公告大厅', 'hn_hnszfcgdzmc_syssqq_ggdt', '430502', announcementTypes, 1),
+    Menu('邵阳市大祥区-公告大厅', 'hn_hnszfcgdzmc_sysdxq_ggdt', '430503', announcementTypes, 1),
+    Menu('邵阳市北塔区-公告大厅', 'hn_hnszfcgdzmc_sysbtq_ggdt', '430511', announcementTypes, 1),
+    Menu('邵阳市邵东市-公告大厅', 'hn_hnszfcgdzmc_syssds_ggdt', '430521', announcementTypes, 3),
+    Menu('邵阳市新邵县-公告大厅', 'hn_hnszfcgdzmc_sysxsx_ggdt', '430522', announcementTypes, 1),
+    Menu('邵阳市邵阳县-公告大厅', 'hn_hnszfcgdzmc_syssyx_ggdt', '430523', announcementTypes, 1),
+    Menu('邵阳市隆回县-公告大厅', 'hn_hnszfcgdzmc_syslhx_ggdt', '430524', announcementTypes, 2),
+    Menu('邵阳市洞口县-公告大厅', 'hn_hnszfcgdzmc_sysdkx_ggdt', '430525', announcementTypes, 1),
+    Menu('邵阳市绥宁县-公告大厅', 'hn_hnszfcgdzmc_syssnx_ggdt', '430527', announcementTypes, 1),
+    Menu('邵阳市新宁县-公告大厅', 'hn_hnszfcgdzmc_sysxnx_ggdt', '430528', announcementTypes, 1),
+    Menu('邵阳市城步苗族自治县-公告大厅', 'hn_hnszfcgdzmc_syscbmzzzx_ggdt', '430529', announcementTypes, 1),
+    Menu('邵阳市邵阳经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_syssyjjkfq_ggdt', '430541', announcementTypes, 1),
+    Menu('邵阳市武冈市-公告大厅', 'hn_hnszfcgdzmc_syswgs_ggdt', '430581', announcementTypes, 1),
+    Menu('邵阳市本级-公告大厅', 'hn_hnszfcgdzmc_sysbj_ggdt', '430599', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-郴州市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 郴州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('郴州市北湖区-公告大厅', 'hn_hnszfcgdzmc_czsbhq_ggdt', '431002', announcementTypes, 1),
+    Menu('郴州市苏仙区-公告大厅', 'hn_hnszfcgdzmc_czssxq_ggdt', '431003', announcementTypes, 1),
+    Menu('郴州市桂阳县-公告大厅', 'hn_hnszfcgdzmc_czsgyx_ggdt', '431021', announcementTypes, 1),
+    Menu('郴州市宜章县-公告大厅', 'hn_hnszfcgdzmc_czsyzx_ggdt', '431022', announcementTypes, 1),
+    Menu('郴州市永兴县-公告大厅', 'hn_hnszfcgdzmc_czsyxx_ggdt', '431023', announcementTypes, 1),
+    Menu('郴州市嘉禾县-公告大厅', 'hn_hnszfcgdzmc_czsjhx_ggdt', '431024', announcementTypes, 1),
+    Menu('郴州市临武县-公告大厅', 'hn_hnszfcgdzmc_czslwx_ggdt', '431025', announcementTypes, 1),
+    Menu('郴州市汝城县-公告大厅', 'hn_hnszfcgdzmc_czsrcx_ggdt', '431026', announcementTypes, 1),
+    Menu('郴州市桂东县-公告大厅', 'hn_hnszfcgdzmc_czsgdx_ggdt', '431027', announcementTypes, 1),
+    Menu('郴州市安仁县-公告大厅', 'hn_hnszfcgdzmc_czsarx_ggdt', '431028', announcementTypes, 1),
+    Menu('郴州市资兴市-公告大厅', 'hn_hnszfcgdzmc_czszxs_ggdt', '431081', announcementTypes, 1),
+    Menu('郴州市本级-公告大厅', 'hn_hnszfcgdzmc_czsbj_ggdt', '431099', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/成交公告-长沙市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 成交公告 - 长沙市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8020, 8025, 8026, 8013]
+
+menus = [
+    Menu('长沙市芙蓉区-公告大厅', 'hn_hnszfcgdzmc_cssfrq_ggdt', '430102', announcementTypes, 1),
+    Menu('长沙市天心区-公告大厅', 'hn_hnszfcgdzmc_csstxq_ggdt', '430103', announcementTypes, 1),
+    Menu('长沙市岳麓区-公告大厅', 'hn_hnszfcgdzmc_cssylq_ggdt', '430104', announcementTypes, 1),
+    Menu('长沙市开福区-公告大厅', 'hn_hnszfcgdzmc_csskfq_ggdt', '430105', announcementTypes, 1),
+    Menu('长沙市雨花区-公告大厅', 'hn_hnszfcgdzmc_cssyhq_ggdt', '430111', announcementTypes, 1),
+    Menu('长沙市望城区-公告大厅', 'hn_hnszfcgdzmc_csswcq_ggdt', '430112', announcementTypes, 1),
+    Menu('长沙市长沙县-公告大厅', 'hn_hnszfcgdzmc_csscsx_ggdt', '430121', announcementTypes, 1),
+    Menu('长沙市湖南湘江新区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csshnxjxqglwyh_ggdt', '430132', announcementTypes, 1),
+    Menu('长沙市浏阳市-公告大厅', 'hn_hnszfcgdzmc_csslys_ggdt', '430181', announcementTypes, 1),
+    Menu('长沙市宁乡市-公告大厅', 'hn_hnszfcgdzmc_cssnxs_ggdt', '430182', announcementTypes, 1),
+    Menu('长沙市长沙高新技术产业开发区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csscsgxqglwyh_ggdt', '430131', announcementTypes, 1),
+    Menu('长沙市本级-公告大厅', 'hn_hnszfcgdzmc_cssbj_ggdt', '430199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/直购定点-湖南省.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 直购定点 - 湖南省
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [808040008]
+
+menus = [
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '43', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-娄底市-湘西土家苗族自治州-省本级.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 娄底市 - 湘西土家族苗族自治州 - 省本级
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('娄底市娄星区-公告大厅', 'hn_hnszfcgdzmc_ldslxq_ggdt', '431302', announcementTypes, 1),
+    Menu('娄底市双峰县-公告大厅', 'hn_hnszfcgdzmc_ldssfx_ggdt', '431321', announcementTypes, 1),
+    Menu('娄底市新化县-公告大厅', 'hn_hnszfcgdzmc_ldsxhx_ggdt', '431322', announcementTypes, 1),
+    Menu('娄底市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_ldsjjjskfq_ggdt', '431331', announcementTypes, 1),
+    Menu('娄底市冷水江市-公告大厅', 'hn_hnszfcgdzmc_ldslsjs_ggdt', '431381', announcementTypes, 1),
+    Menu('娄底市涟源市-公告大厅', 'hn_hnszfcgdzmc_ldslys_ggdt', '431382', announcementTypes, 1),
+    Menu('娄底市本级-公告大厅', 'hn_hnszfcgdzmc_ldsbj_ggdt', '431399', announcementTypes, 1),
+
+    Menu('湘西土家族苗族自治州吉首市-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzjss_ggdt', '433101', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州泸溪县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlxx_ggdt', '433122', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州凤凰县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzfhx_ggdt', '433123', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州花垣县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzhyx_ggdt', '433124', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州保靖县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzbjx_ggdt', '433125', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州古丈县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzgzx_ggdt', '433126', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州永顺县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzysx_ggdt', '433127', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州龙山县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlsx_ggdt', '433130', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州本级-公告大厅', 'hn_hnszfcgdzmc_xxtjmzzzzbj_ggdt', '433199', announcementTypes, 1),
+
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 2),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-岳阳市-常德市.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 岳阳市 - 常德市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('岳阳市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_yysyyjjjskfq_ggdt', '430601', announcementTypes, 1),
+    Menu('岳阳市岳阳楼区-公告大厅', 'hn_hnszfcgdzmc_yysyylq_ggdt', '430602', announcementTypes, 1),
+    Menu('岳阳市云溪区-公告大厅', 'hn_hnszfcgdzmc_yysyxq_ggdt', '430603', announcementTypes, 1),
+    Menu('岳阳市君山区-公告大厅', 'hn_hnszfcgdzmc_yysjsq_ggdt', '430611', announcementTypes, 1),
+    Menu('岳阳市岳阳县-公告大厅', 'hn_hnszfcgdzmc_yysyyx_ggdt', '430621', announcementTypes, 1),
+    Menu('岳阳市华容县-公告大厅', 'hn_hnszfcgdzmc_yyshrx_ggdt', '430623', announcementTypes, 1),
+    Menu('岳阳市湘阴县-公告大厅', 'hn_hnszfcgdzmc_yysxyx_ggdt', '430624', announcementTypes, 1),
+    Menu('岳阳市平江县-公告大厅', 'hn_hnszfcgdzmc_yyspjx_ggdt', '430626', announcementTypes, 1),
+    Menu('岳阳市南湖新区-公告大厅', 'hn_hnszfcgdzmc_yysnhxq_ggdt', '430632', announcementTypes, 1),
+    Menu('岳阳市城陵矶新港区-公告大厅', 'hn_hnszfcgdzmc_yyscljxgq_ggdt', '430634', announcementTypes, 1),
+    Menu('岳阳市屈原管理区-公告大厅', 'hn_hnszfcgdzmc_yysqyglq_ggdt', '430671', announcementTypes, 1),
+    Menu('岳阳市汨罗市-公告大厅', 'hn_hnszfcgdzmc_yysmls_ggdt', '430681', announcementTypes, 1),
+    Menu('岳阳市临湘市-公告大厅', 'hn_hnszfcgdzmc_yyslxs_ggdt', '430682', announcementTypes, 1),
+    Menu('岳阳市本级-公告大厅', 'hn_hnszfcgdzmc_yysbj_ggdt', '430699', announcementTypes, 1),
+
+    Menu('常德市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_cdsjjjskfq_ggdt', '430701', announcementTypes, 1),
+    Menu('常德市武陵区-公告大厅', 'hn_hnszfcgdzmc_cdswlq_ggdt', '430702', announcementTypes, 1),
+    Menu('常德市鼎城区-公告大厅', 'hn_hnszfcgdzmc_cdsdcq_ggdt', '430703', announcementTypes, 1),
+    Menu('常德市柳叶湖旅游度假区-公告大厅', 'hn_hnszfcgdzmc_cdslyhlydjq_ggdt', '430704', announcementTypes, 1),
+    Menu('常德市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_cdsgxjscykfq_ggdt', '430705', announcementTypes, 1),
+    Menu('常德市西湖管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxhglq_ggdt', '430714', announcementTypes, 1),
+    Menu('常德市安乡县-公告大厅', 'hn_hnszfcgdzmc_cdsaxx_ggdt', '430721', announcementTypes, 1),
+    Menu('常德市汉寿县-公告大厅', 'hn_hnszfcgdzmc_cdshsx_ggdt', '430722', announcementTypes, 1),
+    Menu('常德市澧县-公告大厅', 'hn_hnszfcgdzmc_cdslx_ggdt', '430723', announcementTypes, 1),
+    Menu('常德市临澧县-公告大厅', 'hn_hnszfcgdzmc_cdsllx_ggdt', '430724', announcementTypes, 1),
+    Menu('常德市桃源县-公告大厅', 'hn_hnszfcgdzmc_cdstyx_ggdt', '430725', announcementTypes, 1),
+    Menu('常德市石门县-公告大厅', 'hn_hnszfcgdzmc_cdssmx_ggdt', '430726', announcementTypes, 1),
+    Menu('常德市西洞庭管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxdtglq_ggdt', '430771', announcementTypes, 1),
+    Menu('常德市津市市-公告大厅', 'hn_hnszfcgdzmc_cdsjss_ggdt', '430781', announcementTypes, 1),
+    Menu('常德市常德市本级-公告大厅', 'hn_hnszfcgdzmc_cdsbj_ggdt', '430799', announcementTypes, 1),
+]
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-张家界市-郴州市-益阳市.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 张家界市 - 益阳市 - 郴州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('张家界市永定区-公告大厅', 'hn_hnszfcgdzmc_zjjsydq_ggdt', '430802', announcementTypes, 1),
+    Menu('张家界市武陵源区-公告大厅', 'hn_hnszfcgdzmc_zjjswlyq_ggdt', '430811', announcementTypes, 1),
+    Menu('张家界市慈利县-公告大厅', 'hn_hnszfcgdzmc_zjjsclx_ggdt', '430821', announcementTypes, 1),
+    Menu('张家界市桑植县-公告大厅', 'hn_hnszfcgdzmc_zjjsszx_ggdt', '430822', announcementTypes, 1),
+    Menu('张家界市本级-公告大厅', 'hn_hnszfcgdzmc_zjjsbj_ggdt', '430899', announcementTypes, 1),
+
+    Menu('益阳市资阳区-公告大厅', 'hn_hnszfcgdzmc_yyszyq_ggdt', '430902', announcementTypes, 1),
+    Menu('益阳市赫山区-公告大厅', 'hn_hnszfcgdzmc_yyshsq_ggdt', '430903', announcementTypes, 1),
+    Menu('益阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_yysgxjscykfq_ggdt', '430911', announcementTypes, 1),
+    Menu('益阳市大通湖区-公告大厅', 'hn_hnszfcgdzmc_yysdthq_ggdt', '430912', announcementTypes, 1),
+    Menu('益阳市南县-公告大厅', 'hn_hnszfcgdzmc_yysnx_ggdt', '430921', announcementTypes, 1),
+    Menu('益阳市桃江县-公告大厅', 'hn_hnszfcgdzmc_yystjx_ggdt', '430922', announcementTypes, 1),
+    Menu('益阳市安化县-公告大厅', 'hn_hnszfcgdzmc_yysahx_ggdt', '430923', announcementTypes, 1),
+    Menu('益阳市沅江市-公告大厅', 'hn_hnszfcgdzmc_yysyjs_ggdt', '430981', announcementTypes, 1),
+    Menu('益阳市本级-公告大厅', 'hn_hnszfcgdzmc_yiysbj_ggdt', '430999', announcementTypes, 1),
+
+    Menu('郴州市北湖区-公告大厅', 'hn_hnszfcgdzmc_czsbhq_ggdt', '431002', announcementTypes, 1),
+    Menu('郴州市苏仙区-公告大厅', 'hn_hnszfcgdzmc_czssxq_ggdt', '431003', announcementTypes, 1),
+    Menu('郴州市桂阳县-公告大厅', 'hn_hnszfcgdzmc_czsgyx_ggdt', '431021', announcementTypes, 1),
+    Menu('郴州市宜章县-公告大厅', 'hn_hnszfcgdzmc_czsyzx_ggdt', '431022', announcementTypes, 1),
+    Menu('郴州市永兴县-公告大厅', 'hn_hnszfcgdzmc_czsyxx_ggdt', '431023', announcementTypes, 1),
+    Menu('郴州市嘉禾县-公告大厅', 'hn_hnszfcgdzmc_czsjhx_ggdt', '431024', announcementTypes, 1),
+    Menu('郴州市临武县-公告大厅', 'hn_hnszfcgdzmc_czslwx_ggdt', '431025', announcementTypes, 1),
+    Menu('郴州市汝城县-公告大厅', 'hn_hnszfcgdzmc_czsrcx_ggdt', '431026', announcementTypes, 1),
+    Menu('郴州市桂东县-公告大厅', 'hn_hnszfcgdzmc_czsgdx_ggdt', '431027', announcementTypes, 1),
+    Menu('郴州市安仁县-公告大厅', 'hn_hnszfcgdzmc_czsarx_ggdt', '431028', announcementTypes, 1),
+    Menu('郴州市资兴市-公告大厅', 'hn_hnszfcgdzmc_czszxs_ggdt', '431081', announcementTypes, 1),
+    Menu('郴州市本级-公告大厅', 'hn_hnszfcgdzmc_czsbj_ggdt', '431099', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-永州市-怀化市.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 永州市 - 怀化市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('永州市零陵区-公告大厅', 'hn_hnszfcgdzmc_yzsllq_ggdt', '431102', announcementTypes, 1),
+    Menu('永州市冷水滩区-公告大厅', 'hn_hnszfcgdzmc_yzslstq_ggdt', '431103', announcementTypes, 1),
+    Menu('永州市祁阳县-公告大厅', 'hn_hnszfcgdzmc_yzsqyx_ggdt', '431121', announcementTypes, 1),
+    Menu('永州市东安县-公告大厅', 'hn_hnszfcgdzmc_yzsdax_ggdt', '431122', announcementTypes, 1),
+    Menu('永州市双牌县-公告大厅', 'hn_hnszfcgdzmc_yzsspx_ggdt', '431123', announcementTypes, 1),
+    Menu('永州市道县-公告大厅', 'hn_hnszfcgdzmc_yzsdx_ggdt', '431124', announcementTypes, 1),
+    Menu('永州市江永县-公告大厅', 'hn_hnszfcgdzmc_yzsjyx_ggdt', '431125', announcementTypes, 1),
+    Menu('永州市宁远县-公告大厅', 'hn_hnszfcgdzmc_yzsnyx_ggdt', '431126', announcementTypes, 1),
+    Menu('永州市蓝山县-公告大厅', 'hn_hnszfcgdzmc_yzslsx_ggdt', '431127', announcementTypes, 1),
+    Menu('永州市新田县-公告大厅', 'hn_hnszfcgdzmc_yzsxtx_ggdt', '431128', announcementTypes, 1),
+    Menu('永州市江华瑶族自治县-公告大厅', 'hn_hnszfcgdzmc_yzsjhyzzzx_ggdt', '431129', announcementTypes, 1),
+    Menu('永州市永州市本级-公告大厅', 'hn_hnszfcgdzmc_yzsbj_ggdt', '431199', announcementTypes, 1),
+
+    Menu('怀化市鹤城区-公告大厅', 'hn_hnszfcgdzmc_hhshcq_ggdt', '431202', announcementTypes, 1),
+    Menu('怀化市经济开发区-公告大厅', 'hn_hnszfcgdzmc_hhsjjkfq_ggdt', '431212', announcementTypes, 1),
+    Menu('怀化市洪江管理区-公告大厅', 'hn_hnszfcgdzmc_hhshjglq_ggdt', '431213', announcementTypes, 1),
+    Menu('怀化市中方县-公告大厅', 'hn_hnszfcgdzmc_hhszfx_ggdt', '431221', announcementTypes, 1),
+    Menu('怀化市沅陵县-公告大厅', 'hn_hnszfcgdzmc_hhsylx_ggdt', '431222', announcementTypes, 1),
+    Menu('怀化市辰溪县-公告大厅', 'hn_hnszfcgdzmc_hhscxx_ggdt', '431223', announcementTypes, 1),
+    Menu('怀化市溆浦县-公告大厅', 'hn_hnszfcgdzmc_hhsxpx_ggdt', '431224', announcementTypes, 1),
+    Menu('怀化市会同县-公告大厅', 'hn_hnszfcgdzmc_hhshtx_ggdt', '431225', announcementTypes, 1),
+    Menu('怀化市麻阳苗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsmymzzzx_ggdt', '431226', announcementTypes, 1),
+    Menu('怀化市新晃侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsxhdzzzx_ggdt', '431227', announcementTypes, 1),
+    Menu('怀化市芷江侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhszjdzzzx_ggdt', '431228', announcementTypes, 1),
+    Menu('怀化市靖州苗族侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsjzmzdzzzx_ggdt', '431229', announcementTypes, 1),
+    Menu('怀化市通道侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhstddzzzx_ggdt', '431230', announcementTypes, 1),
+    Menu('怀化市洪江市-公告大厅', 'hn_hnszfcgdzmc_hhshjs_ggdt', '431281', announcementTypes, 1),
+    Menu('怀化市本级-公告大厅', 'hn_hnszfcgdzmc_hhsbj_ggdt', '431299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-衡阳市-邵阳市.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 衡阳市 - 邵阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('衡阳市珠晖区-公告大厅', 'hn_hnszfcgdzmc_hyszhq_ggdt', '430405', announcementTypes, 1),
+    Menu('衡阳市雁峰区-公告大厅', 'hn_hnszfcgdzmc_hysyfq_ggdt', '430406', announcementTypes, 1),
+    Menu('衡阳市石鼓区-公告大厅', 'hn_hnszfcgdzmc_hyssgq_ggdt', '430407', announcementTypes, 1),
+    Menu('衡阳市蒸湘区-公告大厅', 'hn_hnszfcgdzmc_hyszxq_ggdt', '430408', announcementTypes, 1),
+    Menu('衡阳市南岳区-公告大厅', 'hn_hnszfcgdzmc_hysnyq_ggdt', '430412', announcementTypes, 1),
+    Menu('衡阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_hysgxjscykfq_ggdt', '430415', announcementTypes, 1),
+    Menu('衡阳市衡阳县-公告大厅', 'hn_hnszfcgdzmc_hyshyx_ggdt', '430421', announcementTypes, 1),
+    Menu('衡阳市衡南县-公告大厅', 'hn_hnszfcgdzmc_hyshnx_ggdt', '430422', announcementTypes, 1),
+    Menu('衡阳市衡山县-公告大厅', 'hn_hnszfcgdzmc_hyshsx_ggdt', '430423', announcementTypes, 1),
+    Menu('衡阳市衡东县-公告大厅', 'hn_hnszfcgdzmc_hyshdx_ggdt', '430424', announcementTypes, 1),
+    Menu('衡阳市祁东县-公告大厅', 'hn_hnszfcgdzmc_hysqdx_ggdt', '430426', announcementTypes, 1),
+    Menu('衡阳市耒阳市-公告大厅', 'hn_hnszfcgdzmc_hyslys_ggdt', '430481', announcementTypes, 1),
+    Menu('衡阳市常宁市-公告大厅', 'hn_hnszfcgdzmc_hyscns_ggdt', '430482', announcementTypes, 1),
+    Menu('衡阳市本级-公告大厅', 'hn_hnszfcgdzmc_hysbj_ggdt', '430499', announcementTypes, 1),
+
+    Menu('邵阳市双清区-公告大厅', 'hn_hnszfcgdzmc_syssqq_ggdt', '430502', announcementTypes, 1),
+    Menu('邵阳市大祥区-公告大厅', 'hn_hnszfcgdzmc_sysdxq_ggdt', '430503', announcementTypes, 1),
+    Menu('邵阳市北塔区-公告大厅', 'hn_hnszfcgdzmc_sysbtq_ggdt', '430511', announcementTypes, 1),
+    Menu('邵阳市邵东市-公告大厅', 'hn_hnszfcgdzmc_syssds_ggdt', '430521', announcementTypes, 1),
+    Menu('邵阳市新邵县-公告大厅', 'hn_hnszfcgdzmc_sysxsx_ggdt', '430522', announcementTypes, 1),
+    Menu('邵阳市邵阳县-公告大厅', 'hn_hnszfcgdzmc_syssyx_ggdt', '430523', announcementTypes, 1),
+    Menu('邵阳市隆回县-公告大厅', 'hn_hnszfcgdzmc_syslhx_ggdt', '430524', announcementTypes, 1),
+    Menu('邵阳市洞口县-公告大厅', 'hn_hnszfcgdzmc_sysdkx_ggdt', '430525', announcementTypes, 1),
+    Menu('邵阳市绥宁县-公告大厅', 'hn_hnszfcgdzmc_syssnx_ggdt', '430527', announcementTypes, 1),
+    Menu('邵阳市新宁县-公告大厅', 'hn_hnszfcgdzmc_sysxnx_ggdt', '430528', announcementTypes, 1),
+    Menu('邵阳市城步苗族自治县-公告大厅', 'hn_hnszfcgdzmc_syscbmzzzx_ggdt', '430529', announcementTypes, 1),
+    Menu('邵阳市邵阳经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_syssyjjkfq_ggdt', '430541', announcementTypes, 1),
+    Menu('邵阳市武冈市-公告大厅', 'hn_hnszfcgdzmc_syswgs_ggdt', '430581', announcementTypes, 1),
+    Menu('邵阳市本级-公告大厅', 'hn_hnszfcgdzmc_sysbj_ggdt', '430599', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/终止公告-长沙市-株洲市-湘潭市.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 终止公告 - 长沙市 - 株洲市 - 湘潭市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [7002, 8022]
+
+menus = [
+    Menu('长沙市芙蓉区-公告大厅', 'hn_hnszfcgdzmc_cssfrq_ggdt', '430102', announcementTypes, 1),
+    Menu('长沙市天心区-公告大厅', 'hn_hnszfcgdzmc_csstxq_ggdt', '430103', announcementTypes, 1),
+    Menu('长沙市岳麓区-公告大厅', 'hn_hnszfcgdzmc_cssylq_ggdt', '430104', announcementTypes, 1),
+    Menu('长沙市开福区-公告大厅', 'hn_hnszfcgdzmc_csskfq_ggdt', '430105', announcementTypes, 1),
+    Menu('长沙市雨花区-公告大厅', 'hn_hnszfcgdzmc_cssyhq_ggdt', '430111', announcementTypes, 1),
+    Menu('长沙市望城区-公告大厅', 'hn_hnszfcgdzmc_csswcq_ggdt', '430112', announcementTypes, 1),
+    Menu('长沙市长沙县-公告大厅', 'hn_hnszfcgdzmc_csscsx_ggdt', '430121', announcementTypes, 1),
+    Menu('长沙市湖南湘江新区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csshnxjxqglwyh_ggdt', '430132', announcementTypes, 1),
+    Menu('长沙市浏阳市-公告大厅', 'hn_hnszfcgdzmc_csslys_ggdt', '430181', announcementTypes, 1),
+    Menu('长沙市宁乡市-公告大厅', 'hn_hnszfcgdzmc_cssnxs_ggdt', '430182', announcementTypes, 1),
+    Menu('长沙市长沙高新技术产业开发区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csscsgxqglwyh_ggdt', '430131', announcementTypes, 1),
+    Menu('长沙市本级-公告大厅', 'hn_hnszfcgdzmc_cssbj_ggdt', '430199', announcementTypes, 1),
+
+    Menu('株洲市荷塘区-公告大厅', 'hn_hnszfcgdzmc_zzshtq_ggdt', '430202', announcementTypes, 1),
+    Menu('株洲市芦淞区-公告大厅', 'hn_hnszfcgdzmc_zzslsq_ggdt', '430203', announcementTypes, 1),
+    Menu('株洲市石峰区-公告大厅', 'hn_hnszfcgdzmc_zzssfq_ggdt', '430204', announcementTypes, 1),
+    Menu('株洲市天元区-公告大厅', 'hn_hnszfcgdzmc_zzstyq_ggdt', '430211', announcementTypes, 1),
+    Menu('株洲市渌口区-公告大厅', 'hn_hnszfcgdzmc_zzslkq_ggdt', '430212', announcementTypes, 1),
+    Menu('株洲市攸县-公告大厅', 'hn_hnszfcgdzmc_zzsyx_ggdt', '430223', announcementTypes, 1),
+    Menu('株洲市茶陵县-公告大厅', 'hn_hnszfcgdzmc_zzsclx_ggdt', '430224', announcementTypes, 1),
+    Menu('株洲市炎陵县-公告大厅', 'hn_hnszfcgdzmc_zzsylx_ggdt', '430225', announcementTypes, 1),
+    Menu('株洲市株洲云龙示范区-公告大厅', 'hn_hnszfcgdzmc_zzszzylsfq_ggdt', '430232', announcementTypes, 1),
+    Menu('株洲市醴陵市-公告大厅', 'hn_hnszfcgdzmc_zzslls_ggdt', '430281', announcementTypes, 1),
+    Menu('株洲市本级-公告大厅', 'hn_hnszfcgdzmc_zzsbj_ggdt', '430299', announcementTypes, 1),
+
+    Menu('湘潭市雨湖区-公告大厅', 'hn_hnszfcgdzmc_xtsyhq_ggdt', '430302', announcementTypes, 1),
+    Menu('湘潭市岳塘区-公告大厅', 'hn_hnszfcgdzmc_xtsytq_ggdt', '430304', announcementTypes, 1),
+    Menu('湘潭市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_xtsgxjscykfq_ggdt', '430311', announcementTypes, 1),
+    Menu('湘潭经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_xtjjjskfq_ggdt', '430312', announcementTypes, 1),
+    Menu('湘潭市湘潭县-公告大厅', 'hn_hnszfcgdzmc_xtsxtx_ggdt', '430321', announcementTypes, 1),
+    Menu('湘潭市湘乡市-公告大厅', 'hn_hnszfcgdzmc_xtsxxs_ggdt', '430381', announcementTypes, 1),
+    Menu('湘潭市韶山市-公告大厅', 'hn_hnszfcgdzmc_xtssss_ggdt', '430382', announcementTypes, 1),
+    Menu('湘潭市本级-公告大厅', 'hn_hnszfcgdzmc_xtsbj_ggdt', '430399', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-娄底市-湘西土家苗族自治州-省本级.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 娄底市 - 湘西土家族苗族自治州 - 省本级
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('娄底市娄星区-公告大厅', 'hn_hnszfcgdzmc_ldslxq_ggdt', '431302', announcementTypes, 1),
+    Menu('娄底市双峰县-公告大厅', 'hn_hnszfcgdzmc_ldssfx_ggdt', '431321', announcementTypes, 1),
+    Menu('娄底市新化县-公告大厅', 'hn_hnszfcgdzmc_ldsxhx_ggdt', '431322', announcementTypes, 1),
+    Menu('娄底市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_ldsjjjskfq_ggdt', '431331', announcementTypes, 1),
+    Menu('娄底市冷水江市-公告大厅', 'hn_hnszfcgdzmc_ldslsjs_ggdt', '431381', announcementTypes, 1),
+    Menu('娄底市涟源市-公告大厅', 'hn_hnszfcgdzmc_ldslys_ggdt', '431382', announcementTypes, 1),
+    Menu('娄底市本级-公告大厅', 'hn_hnszfcgdzmc_ldsbj_ggdt', '431399', announcementTypes, 1),
+
+    Menu('湘西土家族苗族自治州吉首市-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzjss_ggdt', '433101', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州泸溪县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlxx_ggdt', '433122', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州凤凰县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzfhx_ggdt', '433123', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州花垣县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzhyx_ggdt', '433124', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州保靖县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzbjx_ggdt', '433125', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州古丈县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzgzx_ggdt', '433126', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州永顺县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzysx_ggdt', '433127', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州龙山县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlsx_ggdt', '433130', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州本级-公告大厅', 'hn_hnszfcgdzmc_xxtjmzzzzbj_ggdt', '433199', announcementTypes, 1),
+
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-常德市-张家界市.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 常德市 - 张家界市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('常德市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_cdsjjjskfq_ggdt', '430701', announcementTypes, 1),
+    Menu('常德市武陵区-公告大厅', 'hn_hnszfcgdzmc_cdswlq_ggdt', '430702', announcementTypes, 1),
+    Menu('常德市鼎城区-公告大厅', 'hn_hnszfcgdzmc_cdsdcq_ggdt', '430703', announcementTypes, 1),
+    Menu('常德市柳叶湖旅游度假区-公告大厅', 'hn_hnszfcgdzmc_cdslyhlydjq_ggdt', '430704', announcementTypes, 1),
+    Menu('常德市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_cdsgxjscykfq_ggdt', '430705', announcementTypes, 1),
+    Menu('常德市西湖管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxhglq_ggdt', '430714', announcementTypes, 1),
+    Menu('常德市安乡县-公告大厅', 'hn_hnszfcgdzmc_cdsaxx_ggdt', '430721', announcementTypes, 1),
+    Menu('常德市汉寿县-公告大厅', 'hn_hnszfcgdzmc_cdshsx_ggdt', '430722', announcementTypes, 1),
+    Menu('常德市澧县-公告大厅', 'hn_hnszfcgdzmc_cdslx_ggdt', '430723', announcementTypes, 1),
+    Menu('常德市临澧县-公告大厅', 'hn_hnszfcgdzmc_cdsllx_ggdt', '430724', announcementTypes, 1),
+    Menu('常德市桃源县-公告大厅', 'hn_hnszfcgdzmc_cdstyx_ggdt', '430725', announcementTypes, 1),
+    Menu('常德市石门县-公告大厅', 'hn_hnszfcgdzmc_cdssmx_ggdt', '430726', announcementTypes, 1),
+    Menu('常德市西洞庭管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxdtglq_ggdt', '430771', announcementTypes, 1),
+    Menu('常德市津市市-公告大厅', 'hn_hnszfcgdzmc_cdsjss_ggdt', '430781', announcementTypes, 1),
+    Menu('常德市常德市本级-公告大厅', 'hn_hnszfcgdzmc_cdsbj_ggdt', '430799', announcementTypes, 1),
+
+    Menu('张家界市永定区-公告大厅', 'hn_hnszfcgdzmc_zjjsydq_ggdt', '430802', announcementTypes, 1),
+    Menu('张家界市武陵源区-公告大厅', 'hn_hnszfcgdzmc_zjjswlyq_ggdt', '430811', announcementTypes, 1),
+    Menu('张家界市慈利县-公告大厅', 'hn_hnszfcgdzmc_zjjsclx_ggdt', '430821', announcementTypes, 1),
+    Menu('张家界市桑植县-公告大厅', 'hn_hnszfcgdzmc_zjjsszx_ggdt', '430822', announcementTypes, 1),
+    Menu('张家界市本级-公告大厅', 'hn_hnszfcgdzmc_zjjsbj_ggdt', '430899', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-永州市-怀化市.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 永州市 - 怀化市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('永州市零陵区-公告大厅', 'hn_hnszfcgdzmc_yzsllq_ggdt', '431102', announcementTypes, 1),
+    Menu('永州市冷水滩区-公告大厅', 'hn_hnszfcgdzmc_yzslstq_ggdt', '431103', announcementTypes, 1),
+    Menu('永州市祁阳县-公告大厅', 'hn_hnszfcgdzmc_yzsqyx_ggdt', '431121', announcementTypes, 1),
+    Menu('永州市东安县-公告大厅', 'hn_hnszfcgdzmc_yzsdax_ggdt', '431122', announcementTypes, 1),
+    Menu('永州市双牌县-公告大厅', 'hn_hnszfcgdzmc_yzsspx_ggdt', '431123', announcementTypes, 1),
+    Menu('永州市道县-公告大厅', 'hn_hnszfcgdzmc_yzsdx_ggdt', '431124', announcementTypes, 1),
+    Menu('永州市江永县-公告大厅', 'hn_hnszfcgdzmc_yzsjyx_ggdt', '431125', announcementTypes, 1),
+    Menu('永州市宁远县-公告大厅', 'hn_hnszfcgdzmc_yzsnyx_ggdt', '431126', announcementTypes, 1),
+    Menu('永州市蓝山县-公告大厅', 'hn_hnszfcgdzmc_yzslsx_ggdt', '431127', announcementTypes, 1),
+    Menu('永州市新田县-公告大厅', 'hn_hnszfcgdzmc_yzsxtx_ggdt', '431128', announcementTypes, 1),
+    Menu('永州市江华瑶族自治县-公告大厅', 'hn_hnszfcgdzmc_yzsjhyzzzx_ggdt', '431129', announcementTypes, 1),
+    Menu('永州市永州市本级-公告大厅', 'hn_hnszfcgdzmc_yzsbj_ggdt', '431199', announcementTypes, 1),
+
+    Menu('怀化市鹤城区-公告大厅', 'hn_hnszfcgdzmc_hhshcq_ggdt', '431202', announcementTypes, 1),
+    Menu('怀化市经济开发区-公告大厅', 'hn_hnszfcgdzmc_hhsjjkfq_ggdt', '431212', announcementTypes, 1),
+    Menu('怀化市洪江管理区-公告大厅', 'hn_hnszfcgdzmc_hhshjglq_ggdt', '431213', announcementTypes, 1),
+    Menu('怀化市中方县-公告大厅', 'hn_hnszfcgdzmc_hhszfx_ggdt', '431221', announcementTypes, 1),
+    Menu('怀化市沅陵县-公告大厅', 'hn_hnszfcgdzmc_hhsylx_ggdt', '431222', announcementTypes, 1),
+    Menu('怀化市辰溪县-公告大厅', 'hn_hnszfcgdzmc_hhscxx_ggdt', '431223', announcementTypes, 1),
+    Menu('怀化市溆浦县-公告大厅', 'hn_hnszfcgdzmc_hhsxpx_ggdt', '431224', announcementTypes, 1),
+    Menu('怀化市会同县-公告大厅', 'hn_hnszfcgdzmc_hhshtx_ggdt', '431225', announcementTypes, 1),
+    Menu('怀化市麻阳苗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsmymzzzx_ggdt', '431226', announcementTypes, 1),
+    Menu('怀化市新晃侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsxhdzzzx_ggdt', '431227', announcementTypes, 1),
+    Menu('怀化市芷江侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhszjdzzzx_ggdt', '431228', announcementTypes, 1),
+    Menu('怀化市靖州苗族侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsjzmzdzzzx_ggdt', '431229', announcementTypes, 1),
+    Menu('怀化市通道侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhstddzzzx_ggdt', '431230', announcementTypes, 1),
+    Menu('怀化市洪江市-公告大厅', 'hn_hnszfcgdzmc_hhshjs_ggdt', '431281', announcementTypes, 1),
+    Menu('怀化市本级-公告大厅', 'hn_hnszfcgdzmc_hhsbj_ggdt', '431299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-湘潭市-衡阳市.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 湘潭市 - 衡阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('湘潭市雨湖区-公告大厅', 'hn_hnszfcgdzmc_xtsyhq_ggdt', '430302', announcementTypes, 1),
+    Menu('湘潭市岳塘区-公告大厅', 'hn_hnszfcgdzmc_xtsytq_ggdt', '430304', announcementTypes, 1),
+    Menu('湘潭市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_xtsgxjscykfq_ggdt', '430311', announcementTypes, 1),
+    Menu('湘潭经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_xtjjjskfq_ggdt', '430312', announcementTypes, 1),
+    Menu('湘潭市湘潭县-公告大厅', 'hn_hnszfcgdzmc_xtsxtx_ggdt', '430321', announcementTypes, 1),
+    Menu('湘潭市湘乡市-公告大厅', 'hn_hnszfcgdzmc_xtsxxs_ggdt', '430381', announcementTypes, 1),
+    Menu('湘潭市韶山市-公告大厅', 'hn_hnszfcgdzmc_xtssss_ggdt', '430382', announcementTypes, 1),
+    Menu('湘潭市本级-公告大厅', 'hn_hnszfcgdzmc_xtsbj_ggdt', '430399', announcementTypes, 1),
+
+    Menu('衡阳市珠晖区-公告大厅', 'hn_hnszfcgdzmc_hyszhq_ggdt', '430405', announcementTypes, 1),
+    Menu('衡阳市雁峰区-公告大厅', 'hn_hnszfcgdzmc_hysyfq_ggdt', '430406', announcementTypes, 1),
+    Menu('衡阳市石鼓区-公告大厅', 'hn_hnszfcgdzmc_hyssgq_ggdt', '430407', announcementTypes, 1),
+    Menu('衡阳市蒸湘区-公告大厅', 'hn_hnszfcgdzmc_hyszxq_ggdt', '430408', announcementTypes, 1),
+    Menu('衡阳市南岳区-公告大厅', 'hn_hnszfcgdzmc_hysnyq_ggdt', '430412', announcementTypes, 1),
+    Menu('衡阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_hysgxjscykfq_ggdt', '430415', announcementTypes, 1),
+    Menu('衡阳市衡阳县-公告大厅', 'hn_hnszfcgdzmc_hyshyx_ggdt', '430421', announcementTypes, 1),
+    Menu('衡阳市衡南县-公告大厅', 'hn_hnszfcgdzmc_hyshnx_ggdt', '430422', announcementTypes, 1),
+    Menu('衡阳市衡山县-公告大厅', 'hn_hnszfcgdzmc_hyshsx_ggdt', '430423', announcementTypes, 1),
+    Menu('衡阳市衡东县-公告大厅', 'hn_hnszfcgdzmc_hyshdx_ggdt', '430424', announcementTypes, 1),
+    Menu('衡阳市祁东县-公告大厅', 'hn_hnszfcgdzmc_hysqdx_ggdt', '430426', announcementTypes, 1),
+    Menu('衡阳市耒阳市-公告大厅', 'hn_hnszfcgdzmc_hyslys_ggdt', '430481', announcementTypes, 1),
+    Menu('衡阳市常宁市-公告大厅', 'hn_hnszfcgdzmc_hyscns_ggdt', '430482', announcementTypes, 1),
+    Menu('衡阳市本级-公告大厅', 'hn_hnszfcgdzmc_hysbj_ggdt', '430499', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-益阳市-郴州市.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 益阳市 - 郴州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('益阳市资阳区-公告大厅', 'hn_hnszfcgdzmc_yyszyq_ggdt', '430902', announcementTypes, 1),
+    Menu('益阳市赫山区-公告大厅', 'hn_hnszfcgdzmc_yyshsq_ggdt', '430903', announcementTypes, 1),
+    Menu('益阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_yysgxjscykfq_ggdt', '430911', announcementTypes, 1),
+    Menu('益阳市大通湖区-公告大厅', 'hn_hnszfcgdzmc_yysdthq_ggdt', '430912', announcementTypes, 1),
+    Menu('益阳市南县-公告大厅', 'hn_hnszfcgdzmc_yysnx_ggdt', '430921', announcementTypes, 1),
+    Menu('益阳市桃江县-公告大厅', 'hn_hnszfcgdzmc_yystjx_ggdt', '430922', announcementTypes, 1),
+    Menu('益阳市安化县-公告大厅', 'hn_hnszfcgdzmc_yysahx_ggdt', '430923', announcementTypes, 1),
+    Menu('益阳市沅江市-公告大厅', 'hn_hnszfcgdzmc_yysyjs_ggdt', '430981', announcementTypes, 1),
+    Menu('益阳市本级-公告大厅', 'hn_hnszfcgdzmc_yiysbj_ggdt', '430999', announcementTypes, 1),
+
+    Menu('郴州市北湖区-公告大厅', 'hn_hnszfcgdzmc_czsbhq_ggdt', '431002', announcementTypes, 1),
+    Menu('郴州市苏仙区-公告大厅', 'hn_hnszfcgdzmc_czssxq_ggdt', '431003', announcementTypes, 1),
+    Menu('郴州市桂阳县-公告大厅', 'hn_hnszfcgdzmc_czsgyx_ggdt', '431021', announcementTypes, 1),
+    Menu('郴州市宜章县-公告大厅', 'hn_hnszfcgdzmc_czsyzx_ggdt', '431022', announcementTypes, 1),
+    Menu('郴州市永兴县-公告大厅', 'hn_hnszfcgdzmc_czsyxx_ggdt', '431023', announcementTypes, 1),
+    Menu('郴州市嘉禾县-公告大厅', 'hn_hnszfcgdzmc_czsjhx_ggdt', '431024', announcementTypes, 1),
+    Menu('郴州市临武县-公告大厅', 'hn_hnszfcgdzmc_czslwx_ggdt', '431025', announcementTypes, 1),
+    Menu('郴州市汝城县-公告大厅', 'hn_hnszfcgdzmc_czsrcx_ggdt', '431026', announcementTypes, 1),
+    Menu('郴州市桂东县-公告大厅', 'hn_hnszfcgdzmc_czsgdx_ggdt', '431027', announcementTypes, 1),
+    Menu('郴州市安仁县-公告大厅', 'hn_hnszfcgdzmc_czsarx_ggdt', '431028', announcementTypes, 1),
+    Menu('郴州市资兴市-公告大厅', 'hn_hnszfcgdzmc_czszxs_ggdt', '431081', announcementTypes, 1),
+    Menu('郴州市本级-公告大厅', 'hn_hnszfcgdzmc_czsbj_ggdt', '431099', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-邵阳市-岳阳市.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 邵阳市 - 岳阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('邵阳市双清区-公告大厅', 'hn_hnszfcgdzmc_syssqq_ggdt', '430502', announcementTypes, 1),
+    Menu('邵阳市大祥区-公告大厅', 'hn_hnszfcgdzmc_sysdxq_ggdt', '430503', announcementTypes, 1),
+    Menu('邵阳市北塔区-公告大厅', 'hn_hnszfcgdzmc_sysbtq_ggdt', '430511', announcementTypes, 1),
+    Menu('邵阳市邵东市-公告大厅', 'hn_hnszfcgdzmc_syssds_ggdt', '430521', announcementTypes, 1),
+    Menu('邵阳市新邵县-公告大厅', 'hn_hnszfcgdzmc_sysxsx_ggdt', '430522', announcementTypes, 1),
+    Menu('邵阳市邵阳县-公告大厅', 'hn_hnszfcgdzmc_syssyx_ggdt', '430523', announcementTypes, 1),
+    Menu('邵阳市隆回县-公告大厅', 'hn_hnszfcgdzmc_syslhx_ggdt', '430524', announcementTypes, 1),
+    Menu('邵阳市洞口县-公告大厅', 'hn_hnszfcgdzmc_sysdkx_ggdt', '430525', announcementTypes, 1),
+    Menu('邵阳市绥宁县-公告大厅', 'hn_hnszfcgdzmc_syssnx_ggdt', '430527', announcementTypes, 1),
+    Menu('邵阳市新宁县-公告大厅', 'hn_hnszfcgdzmc_sysxnx_ggdt', '430528', announcementTypes, 1),
+    Menu('邵阳市城步苗族自治县-公告大厅', 'hn_hnszfcgdzmc_syscbmzzzx_ggdt', '430529', announcementTypes, 1),
+    Menu('邵阳市邵阳经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_syssyjjkfq_ggdt', '430541', announcementTypes, 1),
+    Menu('邵阳市武冈市-公告大厅', 'hn_hnszfcgdzmc_syswgs_ggdt', '430581', announcementTypes, 1),
+    Menu('邵阳市本级-公告大厅', 'hn_hnszfcgdzmc_sysbj_ggdt', '430599', announcementTypes, 1),
+
+    Menu('岳阳市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_yysyyjjjskfq_ggdt', '430601', announcementTypes, 1),
+    Menu('岳阳市岳阳楼区-公告大厅', 'hn_hnszfcgdzmc_yysyylq_ggdt', '430602', announcementTypes, 1),
+    Menu('岳阳市云溪区-公告大厅', 'hn_hnszfcgdzmc_yysyxq_ggdt', '430603', announcementTypes, 1),
+    Menu('岳阳市君山区-公告大厅', 'hn_hnszfcgdzmc_yysjsq_ggdt', '430611', announcementTypes, 1),
+    Menu('岳阳市岳阳县-公告大厅', 'hn_hnszfcgdzmc_yysyyx_ggdt', '430621', announcementTypes, 1),
+    Menu('岳阳市华容县-公告大厅', 'hn_hnszfcgdzmc_yyshrx_ggdt', '430623', announcementTypes, 1),
+    Menu('岳阳市湘阴县-公告大厅', 'hn_hnszfcgdzmc_yysxyx_ggdt', '430624', announcementTypes, 1),
+    Menu('岳阳市平江县-公告大厅', 'hn_hnszfcgdzmc_yyspjx_ggdt', '430626', announcementTypes, 1),
+    Menu('岳阳市南湖新区-公告大厅', 'hn_hnszfcgdzmc_yysnhxq_ggdt', '430632', announcementTypes, 1),
+    Menu('岳阳市城陵矶新港区-公告大厅', 'hn_hnszfcgdzmc_yyscljxgq_ggdt', '430634', announcementTypes, 1),
+    Menu('岳阳市屈原管理区-公告大厅', 'hn_hnszfcgdzmc_yysqyglq_ggdt', '430671', announcementTypes, 1),
+    Menu('岳阳市汨罗市-公告大厅', 'hn_hnszfcgdzmc_yysmls_ggdt', '430681', announcementTypes, 1),
+    Menu('岳阳市临湘市-公告大厅', 'hn_hnszfcgdzmc_yyslxs_ggdt', '430682', announcementTypes, 1),
+    Menu('岳阳市本级-公告大厅', 'hn_hnszfcgdzmc_yysbj_ggdt', '430699', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/邀请公告-长沙市-株洲市.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 邀请公告 - 长沙市 - 株洲市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [10017, 8018, 8023, 8024]
+
+menus = [
+    Menu('长沙市芙蓉区-公告大厅', 'hn_hnszfcgdzmc_cssfrq_ggdt', '430102', announcementTypes, 1),
+    Menu('长沙市天心区-公告大厅', 'hn_hnszfcgdzmc_csstxq_ggdt', '430103', announcementTypes, 1),
+    Menu('长沙市岳麓区-公告大厅', 'hn_hnszfcgdzmc_cssylq_ggdt', '430104', announcementTypes, 1),
+    Menu('长沙市开福区-公告大厅', 'hn_hnszfcgdzmc_csskfq_ggdt', '430105', announcementTypes, 1),
+    Menu('长沙市雨花区-公告大厅', 'hn_hnszfcgdzmc_cssyhq_ggdt', '430111', announcementTypes, 1),
+    Menu('长沙市望城区-公告大厅', 'hn_hnszfcgdzmc_csswcq_ggdt', '430112', announcementTypes, 1),
+    Menu('长沙市长沙县-公告大厅', 'hn_hnszfcgdzmc_csscsx_ggdt', '430121', announcementTypes, 1),
+    Menu('长沙市湖南湘江新区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csshnxjxqglwyh_ggdt', '430132', announcementTypes, 1),
+    Menu('长沙市浏阳市-公告大厅', 'hn_hnszfcgdzmc_csslys_ggdt', '430181', announcementTypes, 1),
+    Menu('长沙市宁乡市-公告大厅', 'hn_hnszfcgdzmc_cssnxs_ggdt', '430182', announcementTypes, 1),
+    Menu('长沙市长沙高新技术产业开发区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csscsgxqglwyh_ggdt', '430131', announcementTypes, 1),
+    Menu('长沙市本级-公告大厅', 'hn_hnszfcgdzmc_cssbj_ggdt', '430199', announcementTypes, 1),
+
+    Menu('株洲市荷塘区-公告大厅', 'hn_hnszfcgdzmc_zzshtq_ggdt', '430202', announcementTypes, 1),
+    Menu('株洲市芦淞区-公告大厅', 'hn_hnszfcgdzmc_zzslsq_ggdt', '430203', announcementTypes, 1),
+    Menu('株洲市石峰区-公告大厅', 'hn_hnszfcgdzmc_zzssfq_ggdt', '430204', announcementTypes, 1),
+    Menu('株洲市天元区-公告大厅', 'hn_hnszfcgdzmc_zzstyq_ggdt', '430211', announcementTypes, 1),
+    Menu('株洲市渌口区-公告大厅', 'hn_hnszfcgdzmc_zzslkq_ggdt', '430212', announcementTypes, 1),
+    Menu('株洲市攸县-公告大厅', 'hn_hnszfcgdzmc_zzsyx_ggdt', '430223', announcementTypes, 1),
+    Menu('株洲市茶陵县-公告大厅', 'hn_hnszfcgdzmc_zzsclx_ggdt', '430224', announcementTypes, 1),
+    Menu('株洲市炎陵县-公告大厅', 'hn_hnszfcgdzmc_zzsylx_ggdt', '430225', announcementTypes, 1),
+    Menu('株洲市株洲云龙示范区-公告大厅', 'hn_hnszfcgdzmc_zzszzylsfq_ggdt', '430232', announcementTypes, 1),
+    Menu('株洲市醴陵市-公告大厅', 'hn_hnszfcgdzmc_zzslls_ggdt', '430281', announcementTypes, 1),
+    Menu('株洲市本级-公告大厅', 'hn_hnszfcgdzmc_zzsbj_ggdt', '430299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-娄底市.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 娄底市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('娄底市娄星区-公告大厅', 'hn_hnszfcgdzmc_ldslxq_ggdt', '431302', announcementTypes, 1),
+    Menu('娄底市双峰县-公告大厅', 'hn_hnszfcgdzmc_ldssfx_ggdt', '431321', announcementTypes, 1),
+    Menu('娄底市新化县-公告大厅', 'hn_hnszfcgdzmc_ldsxhx_ggdt', '431322', announcementTypes, 1),
+    Menu('娄底市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_ldsjjjskfq_ggdt', '431331', announcementTypes, 1),
+    Menu('娄底市冷水江市-公告大厅', 'hn_hnszfcgdzmc_ldslsjs_ggdt', '431381', announcementTypes, 1),
+    Menu('娄底市涟源市-公告大厅', 'hn_hnszfcgdzmc_ldslys_ggdt', '431382', announcementTypes, 1),
+    Menu('娄底市本级-公告大厅', 'hn_hnszfcgdzmc_ldsbj_ggdt', '431399', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-岳阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 岳阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('岳阳市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_yysyyjjjskfq_ggdt', '430601', announcementTypes, 1),
+    Menu('岳阳市岳阳楼区-公告大厅', 'hn_hnszfcgdzmc_yysyylq_ggdt', '430602', announcementTypes, 1),
+    Menu('岳阳市云溪区-公告大厅', 'hn_hnszfcgdzmc_yysyxq_ggdt', '430603', announcementTypes, 1),
+    Menu('岳阳市君山区-公告大厅', 'hn_hnszfcgdzmc_yysjsq_ggdt', '430611', announcementTypes, 1),
+    Menu('岳阳市岳阳县-公告大厅', 'hn_hnszfcgdzmc_yysyyx_ggdt', '430621', announcementTypes, 1),
+    Menu('岳阳市华容县-公告大厅', 'hn_hnszfcgdzmc_yyshrx_ggdt', '430623', announcementTypes, 1),
+    Menu('岳阳市湘阴县-公告大厅', 'hn_hnszfcgdzmc_yysxyx_ggdt', '430624', announcementTypes, 1),
+    Menu('岳阳市平江县-公告大厅', 'hn_hnszfcgdzmc_yyspjx_ggdt', '430626', announcementTypes, 1),
+    Menu('岳阳市南湖新区-公告大厅', 'hn_hnszfcgdzmc_yysnhxq_ggdt', '430632', announcementTypes, 1),
+    Menu('岳阳市城陵矶新港区-公告大厅', 'hn_hnszfcgdzmc_yyscljxgq_ggdt', '430634', announcementTypes, 1),
+    Menu('岳阳市屈原管理区-公告大厅', 'hn_hnszfcgdzmc_yysqyglq_ggdt', '430671', announcementTypes, 1),
+    Menu('岳阳市汨罗市-公告大厅', 'hn_hnszfcgdzmc_yysmls_ggdt', '430681', announcementTypes, 1),
+    Menu('岳阳市临湘市-公告大厅', 'hn_hnszfcgdzmc_yyslxs_ggdt', '430682', announcementTypes, 1),
+    Menu('岳阳市本级-公告大厅', 'hn_hnszfcgdzmc_yysbj_ggdt', '430699', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-常德市.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 常德市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('常德市经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_cdsjjjskfq_ggdt', '430701', announcementTypes, 1),
+    Menu('常德市武陵区-公告大厅', 'hn_hnszfcgdzmc_cdswlq_ggdt', '430702', announcementTypes, 1),
+    Menu('常德市鼎城区-公告大厅', 'hn_hnszfcgdzmc_cdsdcq_ggdt', '430703', announcementTypes, 1),
+    Menu('常德市柳叶湖旅游度假区-公告大厅', 'hn_hnszfcgdzmc_cdslyhlydjq_ggdt', '430704', announcementTypes, 1),
+    Menu('常德市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_cdsgxjscykfq_ggdt', '430705', announcementTypes, 1),
+    Menu('常德市西湖管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxhglq_ggdt', '430714', announcementTypes, 1),
+    Menu('常德市安乡县-公告大厅', 'hn_hnszfcgdzmc_cdsaxx_ggdt', '430721', announcementTypes, 1),
+    Menu('常德市汉寿县-公告大厅', 'hn_hnszfcgdzmc_cdshsx_ggdt', '430722', announcementTypes, 1),
+    Menu('常德市澧县-公告大厅', 'hn_hnszfcgdzmc_cdslx_ggdt', '430723', announcementTypes, 1),
+    Menu('常德市临澧县-公告大厅', 'hn_hnszfcgdzmc_cdsllx_ggdt', '430724', announcementTypes, 1),
+    Menu('常德市桃源县-公告大厅', 'hn_hnszfcgdzmc_cdstyx_ggdt', '430725', announcementTypes, 1),
+    Menu('常德市石门县-公告大厅', 'hn_hnszfcgdzmc_cdssmx_ggdt', '430726', announcementTypes, 1),
+    Menu('常德市西洞庭管理区-公告大厅', 'hn_hnszfcgdzmc_cdsxdtglq_ggdt', '430771', announcementTypes, 1),
+    Menu('常德市津市市-公告大厅', 'hn_hnszfcgdzmc_cdsjss_ggdt', '430781', announcementTypes, 1),
+    Menu('常德市常德市本级-公告大厅', 'hn_hnszfcgdzmc_cdsbj_ggdt', '430799', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-张家界市.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 张家界市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('张家界市永定区-公告大厅', 'hn_hnszfcgdzmc_zjjsydq_ggdt', '430802', announcementTypes, 1),
+    Menu('张家界市武陵源区-公告大厅', 'hn_hnszfcgdzmc_zjjswlyq_ggdt', '430811', announcementTypes, 1),
+    Menu('张家界市慈利县-公告大厅', 'hn_hnszfcgdzmc_zjjsclx_ggdt', '430821', announcementTypes, 1),
+    Menu('张家界市桑植县-公告大厅', 'hn_hnszfcgdzmc_zjjsszx_ggdt', '430822', announcementTypes, 1),
+    Menu('张家界市本级-公告大厅', 'hn_hnszfcgdzmc_zjjsbj_ggdt', '430899', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-怀化市.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 怀化市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('怀化市鹤城区-公告大厅', 'hn_hnszfcgdzmc_hhshcq_ggdt', '431202', announcementTypes, 1),
+    Menu('怀化市经济开发区-公告大厅', 'hn_hnszfcgdzmc_hhsjjkfq_ggdt', '431212', announcementTypes, 1),
+    Menu('怀化市洪江管理区-公告大厅', 'hn_hnszfcgdzmc_hhshjglq_ggdt', '431213', announcementTypes, 1),
+    Menu('怀化市中方县-公告大厅', 'hn_hnszfcgdzmc_hhszfx_ggdt', '431221', announcementTypes, 1),
+    Menu('怀化市沅陵县-公告大厅', 'hn_hnszfcgdzmc_hhsylx_ggdt', '431222', announcementTypes, 1),
+    Menu('怀化市辰溪县-公告大厅', 'hn_hnszfcgdzmc_hhscxx_ggdt', '431223', announcementTypes, 1),
+    Menu('怀化市溆浦县-公告大厅', 'hn_hnszfcgdzmc_hhsxpx_ggdt', '431224', announcementTypes, 1),
+    Menu('怀化市会同县-公告大厅', 'hn_hnszfcgdzmc_hhshtx_ggdt', '431225', announcementTypes, 1),
+    Menu('怀化市麻阳苗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsmymzzzx_ggdt', '431226', announcementTypes, 1),
+    Menu('怀化市新晃侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsxhdzzzx_ggdt', '431227', announcementTypes, 1),
+    Menu('怀化市芷江侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhszjdzzzx_ggdt', '431228', announcementTypes, 1),
+    Menu('怀化市靖州苗族侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhsjzmzdzzzx_ggdt', '431229', announcementTypes, 1),
+    Menu('怀化市通道侗族自治县-公告大厅', 'hn_hnszfcgdzmc_hhstddzzzx_ggdt', '431230', announcementTypes, 1),
+    Menu('怀化市洪江市-公告大厅', 'hn_hnszfcgdzmc_hhshjs_ggdt', '431281', announcementTypes, 1),
+    Menu('怀化市本级-公告大厅', 'hn_hnszfcgdzmc_hhsbj_ggdt', '431299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-株洲市.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 株洲市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('株洲市荷塘区-公告大厅', 'hn_hnszfcgdzmc_zzshtq_ggdt', '430202', announcementTypes, 1),
+    Menu('株洲市芦淞区-公告大厅', 'hn_hnszfcgdzmc_zzslsq_ggdt', '430203', announcementTypes, 1),
+    Menu('株洲市石峰区-公告大厅', 'hn_hnszfcgdzmc_zzssfq_ggdt', '430204', announcementTypes, 1),
+    Menu('株洲市天元区-公告大厅', 'hn_hnszfcgdzmc_zzstyq_ggdt', '430211', announcementTypes, 1),
+    Menu('株洲市渌口区-公告大厅', 'hn_hnszfcgdzmc_zzslkq_ggdt', '430212', announcementTypes, 1),
+    Menu('株洲市攸县-公告大厅', 'hn_hnszfcgdzmc_zzsyx_ggdt', '430223', announcementTypes, 1),
+    Menu('株洲市茶陵县-公告大厅', 'hn_hnszfcgdzmc_zzsclx_ggdt', '430224', announcementTypes, 1),
+    Menu('株洲市炎陵县-公告大厅', 'hn_hnszfcgdzmc_zzsylx_ggdt', '430225', announcementTypes, 1),
+    Menu('株洲市株洲云龙示范区-公告大厅', 'hn_hnszfcgdzmc_zzszzylsfq_ggdt', '430232', announcementTypes, 1),
+    Menu('株洲市醴陵市-公告大厅', 'hn_hnszfcgdzmc_zzslls_ggdt', '430281', announcementTypes, 1),
+    Menu('株洲市本级-公告大厅', 'hn_hnszfcgdzmc_zzsbj_ggdt', '430299', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-永州市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 永州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('永州市零陵区-公告大厅', 'hn_hnszfcgdzmc_yzsllq_ggdt', '431102', announcementTypes, 1),
+    Menu('永州市冷水滩区-公告大厅', 'hn_hnszfcgdzmc_yzslstq_ggdt', '431103', announcementTypes, 1),
+    Menu('永州市祁阳县-公告大厅', 'hn_hnszfcgdzmc_yzsqyx_ggdt', '431121', announcementTypes, 1),
+    Menu('永州市东安县-公告大厅', 'hn_hnszfcgdzmc_yzsdax_ggdt', '431122', announcementTypes, 1),
+    Menu('永州市双牌县-公告大厅', 'hn_hnszfcgdzmc_yzsspx_ggdt', '431123', announcementTypes, 1),
+    Menu('永州市道县-公告大厅', 'hn_hnszfcgdzmc_yzsdx_ggdt', '431124', announcementTypes, 1),
+    Menu('永州市江永县-公告大厅', 'hn_hnszfcgdzmc_yzsjyx_ggdt', '431125', announcementTypes, 1),
+    Menu('永州市宁远县-公告大厅', 'hn_hnszfcgdzmc_yzsnyx_ggdt', '431126', announcementTypes, 1),
+    Menu('永州市蓝山县-公告大厅', 'hn_hnszfcgdzmc_yzslsx_ggdt', '431127', announcementTypes, 1),
+    Menu('永州市新田县-公告大厅', 'hn_hnszfcgdzmc_yzsxtx_ggdt', '431128', announcementTypes, 1),
+    Menu('永州市江华瑶族自治县-公告大厅', 'hn_hnszfcgdzmc_yzsjhyzzzx_ggdt', '431129', announcementTypes, 1),
+    Menu('永州市永州市本级-公告大厅', 'hn_hnszfcgdzmc_yzsbj_ggdt', '431199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-湘潭市.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 湘潭市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('湘潭市雨湖区-公告大厅', 'hn_hnszfcgdzmc_xtsyhq_ggdt', '430302', announcementTypes, 1),
+    Menu('湘潭市岳塘区-公告大厅', 'hn_hnszfcgdzmc_xtsytq_ggdt', '430304', announcementTypes, 1),
+    Menu('湘潭市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_xtsgxjscykfq_ggdt', '430311', announcementTypes, 1),
+    Menu('湘潭经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_xtjjjskfq_ggdt', '430312', announcementTypes, 1),
+    Menu('湘潭市湘潭县-公告大厅', 'hn_hnszfcgdzmc_xtsxtx_ggdt', '430321', announcementTypes, 1),
+    Menu('湘潭市湘乡市-公告大厅', 'hn_hnszfcgdzmc_xtsxxs_ggdt', '430381', announcementTypes, 1),
+    Menu('湘潭市韶山市-公告大厅', 'hn_hnszfcgdzmc_xtssss_ggdt', '430382', announcementTypes, 1),
+    Menu('湘潭市本级-公告大厅', 'hn_hnszfcgdzmc_xtsbj_ggdt', '430399', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-湘西土家苗族自治州.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 湘西土家族苗族自治州
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('湘西土家族苗族自治州吉首市-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzjss_ggdt', '433101', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州泸溪县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlxx_ggdt', '433122', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州凤凰县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzfhx_ggdt', '433123', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州花垣县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzhyx_ggdt', '433124', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州保靖县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzbjx_ggdt', '433125', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州古丈县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzgzx_ggdt', '433126', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州永顺县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzysx_ggdt', '433127', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州龙山县-公告大厅', 'hn_hnszfcgdzmc_xxtjzmzzzzlsx_ggdt', '433130', announcementTypes, 1),
+    Menu('湘西土家族苗族自治州本级-公告大厅', 'hn_hnszfcgdzmc_xxtjmzzzzbj_ggdt', '433199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-益阳市.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 益阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('益阳市资阳区-公告大厅', 'hn_hnszfcgdzmc_yyszyq_ggdt', '430902', announcementTypes, 1),
+    Menu('益阳市赫山区-公告大厅', 'hn_hnszfcgdzmc_yyshsq_ggdt', '430903', announcementTypes, 1),
+    Menu('益阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_yysgxjscykfq_ggdt', '430911', announcementTypes, 1),
+    Menu('益阳市大通湖区-公告大厅', 'hn_hnszfcgdzmc_yysdthq_ggdt', '430912', announcementTypes, 1),
+    Menu('益阳市南县-公告大厅', 'hn_hnszfcgdzmc_yysnx_ggdt', '430921', announcementTypes, 1),
+    Menu('益阳市桃江县-公告大厅', 'hn_hnszfcgdzmc_yystjx_ggdt', '430922', announcementTypes, 1),
+    Menu('益阳市安化县-公告大厅', 'hn_hnszfcgdzmc_yysahx_ggdt', '430923', announcementTypes, 1),
+    Menu('益阳市沅江市-公告大厅', 'hn_hnszfcgdzmc_yysyjs_ggdt', '430981', announcementTypes, 1),
+    Menu('益阳市本级-公告大厅', 'hn_hnszfcgdzmc_yiysbj_ggdt', '430999', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-省本级.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 省本级
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('湖南省本级-公告大厅', 'hn_hnszfcgdzmc_hnsbj_ggdt', '439900', announcementTypes, 20),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-衡阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 衡阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('衡阳市珠晖区-公告大厅', 'hn_hnszfcgdzmc_hyszhq_ggdt', '430405', announcementTypes, 1),
+    Menu('衡阳市雁峰区-公告大厅', 'hn_hnszfcgdzmc_hysyfq_ggdt', '430406', announcementTypes, 1),
+    Menu('衡阳市石鼓区-公告大厅', 'hn_hnszfcgdzmc_hyssgq_ggdt', '430407', announcementTypes, 1),
+    Menu('衡阳市蒸湘区-公告大厅', 'hn_hnszfcgdzmc_hyszxq_ggdt', '430408', announcementTypes, 1),
+    Menu('衡阳市南岳区-公告大厅', 'hn_hnszfcgdzmc_hysnyq_ggdt', '430412', announcementTypes, 1),
+    Menu('衡阳市高新技术产业开发区-公告大厅', 'hn_hnszfcgdzmc_hysgxjscykfq_ggdt', '430415', announcementTypes, 1),
+    Menu('衡阳市衡阳县-公告大厅', 'hn_hnszfcgdzmc_hyshyx_ggdt', '430421', announcementTypes, 1),
+    Menu('衡阳市衡南县-公告大厅', 'hn_hnszfcgdzmc_hyshnx_ggdt', '430422', announcementTypes, 1),
+    Menu('衡阳市衡山县-公告大厅', 'hn_hnszfcgdzmc_hyshsx_ggdt', '430423', announcementTypes, 1),
+    Menu('衡阳市衡东县-公告大厅', 'hn_hnszfcgdzmc_hyshdx_ggdt', '430424', announcementTypes, 1),
+    Menu('衡阳市祁东县-公告大厅', 'hn_hnszfcgdzmc_hysqdx_ggdt', '430426', announcementTypes, 1),
+    Menu('衡阳市耒阳市-公告大厅', 'hn_hnszfcgdzmc_hyslys_ggdt', '430481', announcementTypes, 1),
+    Menu('衡阳市常宁市-公告大厅', 'hn_hnszfcgdzmc_hyscns_ggdt', '430482', announcementTypes, 1),
+    Menu('衡阳市本级-公告大厅', 'hn_hnszfcgdzmc_hysbj_ggdt', '430499', announcementTypes, 2),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-邵阳市.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 邵阳市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('邵阳市双清区-公告大厅', 'hn_hnszfcgdzmc_syssqq_ggdt', '430502', announcementTypes, 1),
+    Menu('邵阳市大祥区-公告大厅', 'hn_hnszfcgdzmc_sysdxq_ggdt', '430503', announcementTypes, 1),
+    Menu('邵阳市北塔区-公告大厅', 'hn_hnszfcgdzmc_sysbtq_ggdt', '430511', announcementTypes, 1),
+    Menu('邵阳市邵东市-公告大厅', 'hn_hnszfcgdzmc_syssds_ggdt', '430521', announcementTypes, 1),
+    Menu('邵阳市新邵县-公告大厅', 'hn_hnszfcgdzmc_sysxsx_ggdt', '430522', announcementTypes, 1),
+    Menu('邵阳市邵阳县-公告大厅', 'hn_hnszfcgdzmc_syssyx_ggdt', '430523', announcementTypes, 1),
+    Menu('邵阳市隆回县-公告大厅', 'hn_hnszfcgdzmc_syslhx_ggdt', '430524', announcementTypes, 1),
+    Menu('邵阳市洞口县-公告大厅', 'hn_hnszfcgdzmc_sysdkx_ggdt', '430525', announcementTypes, 1),
+    Menu('邵阳市绥宁县-公告大厅', 'hn_hnszfcgdzmc_syssnx_ggdt', '430527', announcementTypes, 1),
+    Menu('邵阳市新宁县-公告大厅', 'hn_hnszfcgdzmc_sysxnx_ggdt', '430528', announcementTypes, 1),
+    Menu('邵阳市城步苗族自治县-公告大厅', 'hn_hnszfcgdzmc_syscbmzzzx_ggdt', '430529', announcementTypes, 1),
+    Menu('邵阳市邵阳经济技术开发区-公告大厅', 'hn_hnszfcgdzmc_syssyjjkfq_ggdt', '430541', announcementTypes, 1),
+    Menu('邵阳市武冈市-公告大厅', 'hn_hnszfcgdzmc_syswgs_ggdt', '430581', announcementTypes, 1),
+    Menu('邵阳市本级-公告大厅', 'hn_hnszfcgdzmc_sysbj_ggdt', '430599', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-郴州市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 郴州市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('郴州市北湖区-公告大厅', 'hn_hnszfcgdzmc_czsbhq_ggdt', '431002', announcementTypes, 1),
+    Menu('郴州市苏仙区-公告大厅', 'hn_hnszfcgdzmc_czssxq_ggdt', '431003', announcementTypes, 1),
+    Menu('郴州市桂阳县-公告大厅', 'hn_hnszfcgdzmc_czsgyx_ggdt', '431021', announcementTypes, 1),
+    Menu('郴州市宜章县-公告大厅', 'hn_hnszfcgdzmc_czsyzx_ggdt', '431022', announcementTypes, 1),
+    Menu('郴州市永兴县-公告大厅', 'hn_hnszfcgdzmc_czsyxx_ggdt', '431023', announcementTypes, 1),
+    Menu('郴州市嘉禾县-公告大厅', 'hn_hnszfcgdzmc_czsjhx_ggdt', '431024', announcementTypes, 1),
+    Menu('郴州市临武县-公告大厅', 'hn_hnszfcgdzmc_czslwx_ggdt', '431025', announcementTypes, 1),
+    Menu('郴州市汝城县-公告大厅', 'hn_hnszfcgdzmc_czsrcx_ggdt', '431026', announcementTypes, 1),
+    Menu('郴州市桂东县-公告大厅', 'hn_hnszfcgdzmc_czsgdx_ggdt', '431027', announcementTypes, 1),
+    Menu('郴州市安仁县-公告大厅', 'hn_hnszfcgdzmc_czsarx_ggdt', '431028', announcementTypes, 1),
+    Menu('郴州市资兴市-公告大厅', 'hn_hnszfcgdzmc_czszxs_ggdt', '431081', announcementTypes, 1),
+    Menu('郴州市本级-公告大厅', 'hn_hnszfcgdzmc_czsbj_ggdt', '431099', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/hnszfcgdzmc/验收公告-长沙市.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-01-24
+---------
+@summary: 验收公告 - 长沙市
+---------
+@author: Lzz
+"""
+from collections import namedtuple
+from spider import Crawl_Hndzmc
+
+Menu = namedtuple('Menu', ['channel', 'spidercode', 'district', 'anTypes', 'crawl_page'])
+
+announcementTypes = [8016, 8021]
+
+menus = [
+    Menu('长沙市芙蓉区-公告大厅', 'hn_hnszfcgdzmc_cssfrq_ggdt', '430102', announcementTypes, 1),
+    Menu('长沙市天心区-公告大厅', 'hn_hnszfcgdzmc_csstxq_ggdt', '430103', announcementTypes, 1),
+    Menu('长沙市岳麓区-公告大厅', 'hn_hnszfcgdzmc_cssylq_ggdt', '430104', announcementTypes, 1),
+    Menu('长沙市开福区-公告大厅', 'hn_hnszfcgdzmc_csskfq_ggdt', '430105', announcementTypes, 1),
+    Menu('长沙市雨花区-公告大厅', 'hn_hnszfcgdzmc_cssyhq_ggdt', '430111', announcementTypes, 1),
+    Menu('长沙市望城区-公告大厅', 'hn_hnszfcgdzmc_csswcq_ggdt', '430112', announcementTypes, 1),
+    Menu('长沙市长沙县-公告大厅', 'hn_hnszfcgdzmc_csscsx_ggdt', '430121', announcementTypes, 1),
+    Menu('长沙市湖南湘江新区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csshnxjxqglwyh_ggdt', '430132', announcementTypes, 1),
+    Menu('长沙市浏阳市-公告大厅', 'hn_hnszfcgdzmc_csslys_ggdt', '430181', announcementTypes, 1),
+    Menu('长沙市宁乡市-公告大厅', 'hn_hnszfcgdzmc_cssnxs_ggdt', '430182', announcementTypes, 1),
+    Menu('长沙市长沙高新技术产业开发区管理委员会-公告大厅', 'hn_hnszfcgdzmc_csscsgxqglwyh_ggdt', '430131', announcementTypes, 1),
+    Menu('长沙市本级-公告大厅', 'hn_hnszfcgdzmc_cssbj_ggdt', '430199', announcementTypes, 1),
+]
+
+# Crawl_Hndzmc().start_list(menus)

lzz_theme/jsxmhjyxdjbbaxt/det_start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ps -ef |grep "jsxmhjyx_details.py" |grep -v grep |awk '{print $2}' |xargs kill -9
+nohup python3 jsxmhjyx_details.py > log/jsxmhjyx_details.out 2>&1 &
+

lzz_theme/jsxmhjyxdjbbaxt/jsxmhjyx_details.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-04-18
+---------
+@summary: 建设项目环境影响登记表备案系统 - 详情页
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from parsel import Selector
+from threading import Timer
+from utils.tools import *
+import warnings
+warnings.filterwarnings('ignore')
+
+requests.packages.urllib3.disable_warnings()
+requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
+try:
+    requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
+except AttributeError:
+    pass
+
+
+class Details:
+
+    def __init__(self):
+        self.db_table = Mongo_client().py_spider
+        self.db_name = self.db_table.theme_list
+        self.zt_details = self.db_table.data_bak
+        # self.proxy = get_proxy()
+        self.rds = Redis_client()
+        self.redis_key = "ztpc_jsxmhjyxdjbbaxt_msg"
+        self.delete_key = ""
+        self.end_state = False
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        }
+
+
+    def detail_get(self, res, item):
+        response = Selector(res.text)
+
+        html = response.xpath('//table[@id="inputFormTable"]').extract_first()  # 标书详细内容
+
+        if "查询超过每日规定次数" in res.text or "IP因非法访问被冻结" in res.text or item['projectname'] not in html:
+            raise EOFError("虚假请求")
+
+        project_person = response.xpath('//label[contains(text(),"联系人")]/../following-sibling::td[1]/text()').extract_first('').strip()
+        if project_person:
+            item['project_person'] = project_person
+        project_phone = response.xpath('//label[contains(text(),"联系电话")]/../following-sibling::td[1]/text()').extract_first('').strip()
+        if project_phone:
+            item['project_phone'] = project_phone
+
+        project_scale = response.xpath('//label[contains(text(),"建设内容及规模")]/../following-sibling::td[1]/text()').extract_first('').strip()
+
+        if project_scale:
+            construction_area = search('[总]*\S*建筑[面|面积]*[约|为]*(.*?)[。|，|,|;]', project_scale)
+            floor_area = search('[总]*\S*占地[面|面积]*[约|为]*(.*?)[。|，|,|;]', project_scale)
+            if not construction_area:
+                construction_area = ""
+            else:
+                construction_area = re.sub(":|：", "", construction_area)
+
+            if not floor_area:
+                floor_area = ""
+            else:
+                floor_area = re.sub(":|：", "", floor_area)
+
+            item['project_scale'] = project_scale
+            item['project_scale_info'] = {
+                "construction_area": construction_area,
+                "floor_area": floor_area,
+            }  # 建设规模及主要内容
+
+        item['contenthtml'] = html
+
+        item = format_fileds_njpc(item)
+
+        try:
+            self.zt_details.insert_one(item)
+            logger.info(f"[采集成功]{item['title']}-{item['publishtime']}")
+        except DuplicateKeyError:
+            logger.info(f"[重复采集]{item['title']}-{item['publishtime']}")
+
+    def fetch_request(self, item):
+
+        response = requests.post(url=item.get("parse_url"), headers=self.headers, proxies=get_proxy(),
+                                 timeout=30, verify=False)
+        time.sleep(random.randint(2,5))
+        return response
+
+    def deal_request(self, item):
+
+        retry_times = 0
+        org_item = item.copy()
+        while retry_times < 5:
+            try:
+                response = self.fetch_request(item)
+                res_code = response.status_code
+                if response is not None and res_code == 200:
+                    self.detail_get(response, item=item)
+                    return True
+                else:
+                    retry_times += 1
+                    time.sleep(1)
+            except Exception as e:
+                item = org_item
+                logger.error(f"{item['href']} 采集异常：{e}")
+                retry_times += 1
+                # time.sleep(random.randint(3, 6))
+                # self.proxy = get_proxy()
+
+        logger.warning(f"[采集失败]{item['href']}")
+        return False
+
+    def countSec(self):
+        for count in range(5, 0, -1):
+            print(f'\r{count} 秒 后结束任务', end='')
+            time.sleep(1)
+        print('\r任务结束')
+
+    def de_redis_key(self):
+        self.end_state = True
+        self.rds.hdel(self.redis_key, self.delete_key)
+        logger.warning("当前数据未采集成功，数据已回填！")
+        self.countSec()
+
+    def start(self, limit=1):
+        logger.debug("********** 详情页采集开始 **********")
+        ts = Timer(1195, self.de_redis_key)  # 声明一个定时器,设置多少s后执行
+        ts.start()
+        with self.db_name.find({"parser_name": "ztpc_jsxmhjyxdjbbaxt", "retry": {"$lt": 3},
+                                "failed": False, "is_crawl": False},sort=[('publishtime',-1)]).limit(limit) as cursor:
+            count = 0
+            for item in cursor:
+                # logger.debug(item)
+                update_id = item["_id"]
+                retry = item["retry"]
+                if self.end_state:
+                    break
+                # if count >= limit:
+                #     break
+                unicode_key = md5value(item.get('href') + item.get('projectname',''))
+                if not self.rds.hexists(self.redis_key, unicode_key):
+                    self.rds.hset(self.redis_key, unicode_key, '')
+                    self.delete_key = unicode_key
+                    count += 1
+                    result = self.deal_request(item)
+                    if result:
+                        self.db_name.update_one({"_id": update_id}, {"$set": {"is_crawl": True}})
+                    else:
+                        retry += 1
+                        self.db_name.update_one({"_id": update_id}, {"$set": {"failed": True, "retry": retry}})
+                        self.rds.hdel(self.redis_key, unicode_key)
+                    time.sleep(random.random())
+
+        logger.debug("********** 详情页采集结束 **********")
+        ts.cancel()  # 脚本规定时间内正常结束，取消定时器
+
+
+if __name__ == "__main__":
+    Details().start(limit=100)

lzz_theme/jsxmhjyxdjbbaxt/jsxmhjyx_list.py

@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-04-18
+---------
+@summary: 建设项目环境影响登记表备案系统 - 列表页
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from concurrent.futures import ThreadPoolExecutor, wait
+from collections import deque
+from utils.RedisDB import RedisFilter
+from utils.tools import *
+from parsel import Selector
+import warnings
+
+warnings.filterwarnings('ignore')
+
+
+requests.packages.urllib3.disable_warnings()
+requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
+try:
+    requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
+except AttributeError:
+    pass
+
+
+
+class Crawl:
+
+    def __init__(self):
+        self.client = Mongo_client().py_spider
+        self.zb_list = self.client.theme_list
+        self.jsxmhjyx_tasks = self.client.jsxmhjyx_tasks
+        self.RDS = RedisFilter()
+        self.real_cont = 0
+        self.order = 0
+        self.proxies_list = []
+        self.dlst = None
+
+    def add_proxies(self, base_proxy_list):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Authorization": "Basic amlhbnl1MDAxOjEyM3F3ZSFB",
+            "Cache-Control": "no-cache",
+            "Pragma": "no-cache",
+            "Proxy-Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+        }
+        url = "http://cc.spdata.jianyu360.com/crawl/proxy/query"
+        response = requests.get(url, headers=headers, timeout=10, verify=False)
+        count = 0
+        ip_list = response.json().get('data')
+        for info in ip_list:
+            proxies = info.get('proxies')
+            match = re.findall(f'{str(proxies)}', str(base_proxy_list))
+            if not match:
+                count += 1
+                base_proxy_list.append(proxies)
+        logger.debug(f"新增代理个数: {count} ,现有代理个数: {len(base_proxy_list)}")
+        return base_proxy_list
+
+    def get_cd(self, proxies=False):
+
+        session = requests.session()
+        session.proxies = proxies
+        headers = {
+            "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        }
+
+        url = "https://beian.china-eia.com/servlet/validateCodeServlet"
+        res = session.get(url, headers=headers, timeout=30, verify=False)
+        code = simple_captcha(res.content)
+
+        headers = {
+            "Accept": "*/*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+        url = "https://beian.china-eia.com/servlet/validateCodeServlet"
+        params = {
+            "validateCode": f"{code}"
+        }
+        resp = session.get(url, headers=headers, params=params, timeout=30, verify=False)
+        if "false" in resp.text:
+            raise ValueError("验证码错误！")
+        else:
+            ck = session.cookies.get_dict()
+            return {"code": code, "ck": ck}
+
+    def fetch_list_page(self, page, keyword, proxies):
+
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://beian.china-eia.com",
+            "Pragma": "no-cache",
+            "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        }
+
+        url = "https://beian.china-eia.com/f/announcement/announcementShow"
+        cf = self.get_cd(proxies)
+        data = {
+            "pageNo": f"{page}",
+            "pageSize": "20",
+            "projectStatus": "1",
+            "projectName": keyword,
+            "searchName": "",
+            "buildProvince": "",
+            "buildCity": "",
+            "buildCounty": "",
+            "recordNumber": "",
+            "validateCode": cf.get('code')
+        }
+        request_params = {
+            "headers": headers,
+            "data": data,
+            "cookies": cf.get('ck'),
+            "proxies": proxies,
+            "timeout": 30,
+            "verify": False,
+        }
+
+        resp = requests.post(url, **request_params)
+        return resp
+
+    def parser_list_page(self, response, idl, proxies):
+        res = Selector(response.text)
+        verify = res.xpath('//div[@id="jbox"]').extract_first()
+        if verify or "查询超过每日规定次数" in response.text or "IP因非法访问被冻结" in response.text:
+            raise EOFError("验证失效")
+        info_list = res.xpath('//table[@id="index_table"]/tbody/tr') or []
+        results_list = []
+        last_date = ""
+        first_date = ""
+        max_page = 1
+        for info in info_list:
+            href = "https://beian.china-eia.com" + info.xpath('./td[last()-1]/a/@href').extract_first()
+            projectname = info.xpath('./td[2]/text()').extract_first("").strip()
+            publish_time = info.xpath('./td[6]/text()').extract_first("").strip()
+            owner = info.xpath('./td[4]/text()').extract_first("").strip()
+            projectaddr = info.xpath('./td[5]/text()').extract_first("").strip()
+
+            item = {
+                "site": "建设项目环境影响登记表备案系统",
+                "channel": "公告",
+                "spidercode": "a_jsxmhjyxdjbbaxt_gg_nipc",
+                "area": "全国",
+                "city": "",
+                "district": "",
+                "href": href,
+                "projectname": projectname,
+                "publishtime": publish_time,
+                "parse_url": href,
+                "owner": owner,
+                "projectaddr": projectaddr,
+                "parser_name": "ztpc_jsxmhjyxdjbbaxt",
+                "is_mixed": False,
+                "is_theme": True,
+                "retry": 0,
+                "comeintime": int2long(time.time()),
+                "is_crawl": False,
+                "failed": False,
+                "sendflag": "false",
+                "T": "bidding",
+            }
+            dedup = [href, projectname]
+            if not self.RDS.data_filter(dedup):
+                results_list.append(item)
+                self.zb_list.insert_one(item)
+                self.RDS.data_save_redis(dedup)
+            last_date = publish_time
+            if not first_date:
+                first_date = publish_time
+
+        if get_today_of_day() in last_date:
+            max_page = 2
+
+        self.jsxmhjyx_tasks.update_one({"_id": idl["_id"]},
+                                       {"$set": {"max_page": max_page,
+                                                 "last_info_date": last_date,
+                                                 "first_info_date": first_date,
+                                                 "state": "finished",
+                                                 "ip": proxies,
+                                                 "updatetime": get_current_date(),
+                                                 "is_crawl": True, }})
+
+        self.real_cont += len(results_list)
+        return len(results_list), len(info_list)
+
+    def crawl_spider(self, page, idl):
+        self.order += 1
+        retey = 0
+        proxies = None
+        keyword = idl['keyword']
+
+        if not self.proxies_list or len(self.dlst) < 5:
+            self.proxies_list = self.add_proxies(self.proxies_list)
+            self.dlst = deque(self.proxies_list)
+
+        while retey < 2:
+            try:
+                proxies = self.dlst.popleft()
+                response = self.fetch_list_page(page, keyword, proxies)
+                logger.debug(f" +++ 关键词：{keyword} 状态码:{response} +++ < {self.order} > ")
+                if response.status_code == 200:
+                    real, fact = self.parser_list_page(response, idl, proxies)
+                    self.dlst.append(proxies)
+                    logger.info(f" --- 关键词：{keyword} - 检索：{fact} 条 - 入库：{real} 条 --- ")
+                    logger.info(f"当前已采集 {self.real_cont} 条数据")
+                    time.sleep(random.randint(2,5))
+                    return
+            except Exception as e:
+                logger.error(f"采集异常：{e}")
+                retey += 1
+
+        self.jsxmhjyx_tasks.update_one({"_id": idl["_id"]},
+                                       {"$set": {"state": "error",
+                                                 "ip": proxies,
+                                                 "updatetime": get_current_date()}})
+        logger.warning(f" --- 关键词：{keyword} 采集失败 --- ")
+
+    def crawl_list_spider(self, page, start_page, works):
+
+        not_crawl = 0
+        interface_data_list = []
+        # 获取采集任务, 检测上一轮是否全部采集完成，并重置采集状态  3338
+        # with self.jsxmhjyx_tasks.find({"max_page": start_page}, no_cursor_timeout=True) as cursor:
+        with self.jsxmhjyx_tasks.find(no_cursor_timeout=True) as cursor:
+            for item in cursor:
+                if item["is_crawl"] == True:
+                    self.jsxmhjyx_tasks.update_one({"_id": item["_id"]}, {"$set": {"is_crawl": False}})
+                else:
+                    not_crawl += 1
+                interface_data_list.append(item)
+            logger.warning(f" *** 上一轮[未]采集关键词数量：{not_crawl} 个 *** ")
+        time.sleep(3)
+
+        with ThreadPoolExecutor(max_workers=works) as excutor:
+            futures = []
+            for item in interface_data_list:
+                # logger.debug(item)
+                future = excutor.submit(self.crawl_spider, page, item)
+                futures.append(future)
+            wait(futures)
+
+    def start_list(self, start_page=1, end_page=1, works=1):
+        logger.debug("********** 列表页开始 **********")
+        for page in range(start_page, end_page + 1):
+            self.crawl_list_spider(page, start_page, works)
+        logger.debug("********** 列表页结束 **********")
+
+
+if __name__ == '__main__':
+    Crawl().start_list(start_page=1, end_page=1, works=1)

lzz_theme/jsxmhjyxdjbbaxt/jsxmhjyx_retry_list.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024-04-09
+---------
+@summary: 建设项目环境影响登记表备案系统 - 列表页 - 采集失败重试
+---------
+@author: Lzz
+"""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.getcwd()))
+from utils.RedisDB import RedisFilter
+from utils.tools import *
+from parsel import Selector
+import warnings
+
+warnings.filterwarnings('ignore')
+from concurrent.futures import ThreadPoolExecutor, wait
+
+requests.packages.urllib3.disable_warnings()
+requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
+try:
+    requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
+except AttributeError:
+    pass
+
+
+class Crawl_Yns:
+
+    def __init__(self):
+        self.client = Mongo_client().py_spider
+        self.zb_list = self.client.theme_list
+        self.jsxmhjyx_tasks = self.client.jsxmhjyx_tasks
+        self.RDS = RedisFilter()
+        self.real_cont = 0
+        self.order = 0
+        self.proxy = get_proxy()
+
+    def get_cd(self, proxies=False):
+
+        session = requests.session()
+        session.proxies = proxies
+        headers = {
+            "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        }
+
+        url = "https://beian.china-eia.com/servlet/validateCodeServlet"
+        count = 0
+        while count < 3:
+            res = session.get(url, headers=headers, timeout=30, verify=False)
+            code = simple_captcha(res.content)
+
+            headers = {
+                "Accept": "*/*",
+                "Accept-Language": "zh-CN,zh;q=0.9",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+                "Pragma": "no-cache",
+                "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+                "X-Requested-With": "XMLHttpRequest",
+            }
+
+            url = "https://beian.china-eia.com/servlet/validateCodeServlet"
+            params = {
+                "validateCode": f"{code}"
+            }
+            resp = session.get(url, headers=headers, params=params, timeout=30, verify=False)
+            if "false" in resp.text:
+                count += 1
+                time.sleep(3)
+            else:
+                ck = session.cookies.get_dict()
+                return {"code": code, "ck": ck}
+
+        raise ValueError("验证码错误！")
+
+    def fetch_list_page(self, page, keyword):
+
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://beian.china-eia.com",
+            "Pragma": "no-cache",
+            "Referer": "https://beian.china-eia.com/f/announcement/announcementShow",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        }
+
+        url = "https://beian.china-eia.com/f/announcement/announcementShow"
+        cf = self.get_cd(self.proxy)
+        data = {
+            "pageNo": f"{page}",
+            "pageSize": "20",
+            "projectStatus": "1",
+            "projectName": keyword,
+            "searchName": "",
+            "buildProvince": "",
+            "buildCity": "",
+            "buildCounty": "",
+            "recordNumber": "",
+            "validateCode": cf.get('code')
+        }
+        request_params = {
+            "headers": headers,
+            "data": data,
+            "cookies": cf.get('ck'),
+            "proxies": self.proxy,
+            "timeout": 60,
+            "verify": False,
+        }
+
+        resp = requests.post(url, **request_params)
+        time.sleep(1)
+        return resp
+
+    def parser_list_page(self, response, idl):
+        res = Selector(response.text)
+        verify = res.xpath('//div[@id="jbox"]').extract_first()
+        if verify or "查询超过每日规定次数" in response.text or "IP因非法访问被冻结" in response.text:
+            raise EOFError("验证失效")
+        info_list = res.xpath('//table[@id="index_table"]/tbody/tr') or []
+        results_list = []
+        last_date = ""
+        max_page = 1
+        for info in info_list:
+            href = "https://beian.china-eia.com" + info.xpath('./td[last()-1]/a/@href').extract_first()
+            projectname = info.xpath('./td[2]/text()').extract_first("").strip()
+            publish_time = info.xpath('./td[6]/text()').extract_first("").strip()
+            owner = info.xpath('./td[4]/text()').extract_first("").strip()
+            projectaddr = info.xpath('./td[5]/text()').extract_first("").strip()
+
+            item = {
+                "site": "建设项目环境影响登记表备案系统",
+                "channel": "公告",
+                "spidercode": "a_jsxmhjyxdjbbaxt_gg_nipc",
+                "area": "全国",
+                "city": "",
+                "district": "",
+                "href": href,
+                "projectname": projectname,
+                "publishtime": publish_time,
+                "parse_url": href,
+                "owner": owner,
+                "projectaddr": projectaddr,
+                "parser_name": "ztpc_jsxmhjyxdjbbaxt",
+                "is_mixed": False,
+                "is_theme": True,
+                "retry": 0,
+                "comeintime": int2long(time.time()),
+                "is_crawl": False,
+                "failed": False,
+                "sendflag": "false",
+                "T": "bidding",
+            }
+            dedup = [href, projectname]
+            if not self.RDS.data_filter(dedup):
+                results_list.append(item)
+                self.zb_list.insert_one(item)
+                self.RDS.data_save_redis(dedup)
+            last_date = publish_time
+
+        if get_today_of_day() in last_date:
+            max_page = 2
+
+        self.jsxmhjyx_tasks.update_one({"_id": idl["_id"]},
+                                       {"$set": {"max_page": max_page,
+                                                 "last_info_date": last_date,
+                                                 "state": "finished",
+                                                 "ip": self.proxy,
+                                                 "updatetime": get_current_date(),
+                                                 "is_crawl": True, }})
+
+        self.real_cont += len(results_list)
+        return len(results_list), len(info_list)
+
+    def crawl_spider(self, page, idl):
+        self.order += 1
+        keyword = idl['keyword']
+        retry = 0
+        while retry < 5:
+            try:
+                response = self.fetch_list_page(page, keyword)
+                logger.debug(f" +++ 关键词：{keyword} 状态码:{response} +++ < {self.order} > ")
+                if response.status_code == 200:
+                    real, fact = self.parser_list_page(response, idl)
+                    logger.info(f" --- 关键词：{keyword} - 检索：{fact} 条 - 入库：{real} 条 --- ")
+                    logger.info(f"当前已采集 {self.real_cont} 条数据")
+                    break
+                else:
+                    self.proxy = get_proxy()
+                    retry += 1
+                    time.sleep(1)
+            except Exception as e:
+                logger.error(f"采集异常：{e}")
+                self.proxy = get_proxy()
+                retry += 1
+                time.sleep(3)
+
+        if retry == 5:
+            self.jsxmhjyx_tasks.update_one({"_id": idl["_id"]},
+                                           {"$set": {"state": "error",
+                                                     "ip": self.proxy,
+                                                     "updatetime": get_current_date()}})
+
+    def crawl_list_spider(self, page, works):
+
+        interface_data_list = []
+        # 获取采集失败的任务
+        with self.jsxmhjyx_tasks.find({"last_info_date": ""}) as cursor:
+            for item in cursor:
+                interface_data_list.append(item)
+        time.sleep(3)
+
+        with ThreadPoolExecutor(max_workers=works) as excutor:
+            futures = []
+            for item in interface_data_list:
+                # logger.debug(item)
+                future = excutor.submit(self.crawl_spider, page, item)
+                futures.append(future)
+            wait(futures)
+
+    def start_list(self, start_page=1, end_page=1, works=1):
+        logger.debug("********** 列表页开始 **********")
+        for page in range(start_page, end_page + 1):
+            self.crawl_list_spider(page, works)
+        logger.debug("********** 列表页结束 **********")
+
+
+if __name__ == '__main__':
+    Crawl_Yns().start_list(start_page=1, end_page=1, works=2)
+