first commit

.idea/manhua.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="projectConfiguration" value="Twisted Trial" />
+    <option name="PROJECT_TEST_RUNNER" value="Twisted Trial" />
+  </component>
+</module>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (gupiao3)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/manhua.iml" filepath="$PROJECT_DIR$/.idea/manhua.iml" />
+    </modules>
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,364 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="ecf921c8-b00f-4c0f-a340-47b61c34fcea" name="Default" comment="" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="TRACKING_ENABLED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+      <file leaf-file-name="settings.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="1782">
+              <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="kukukkk.py" pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="27">
+              <caret line="20" column="23" lean-forward="false" selection-start-line="20" selection-start-column="23" selection-end-line="20" selection-end-column="23" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="pipelines.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="1161">
+              <caret line="43" column="17" lean-forward="false" selection-start-line="41" selection-start-column="8" selection-end-line="43" selection-end-column="17" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>/comiclist</find>
+    </findStrings>
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/manhua/pipelines.py" />
+        <option value="$PROJECT_DIR$/manhua/settings.py" />
+        <option value="$PROJECT_DIR$/manhua/spiders/kukukkk.py" />
+      </list>
+    </option>
+  </component>
+  <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsGulpfileManager">
+    <detection-done>true</detection-done>
+    <sorting>DEFINITION_ORDER</sorting>
+  </component>
+  <component name="ProjectFrameBounds" extendedState="6">
+    <option name="x" value="-8" />
+    <option name="y" value="-8" />
+    <option name="width" value="1382" />
+    <option name="height" value="744" />
+  </component>
+  <component name="ProjectView">
+    <navigator currentView="ProjectPane" proportions="" version="1">
+      <flattenPackages />
+      <showMembers />
+      <showModules />
+      <showLibraryContents />
+      <hideEmptyPackages />
+      <abbreviatePackageNames />
+      <autoscrollToSource />
+      <autoscrollFromSource />
+      <sortByType />
+      <manualOrder />
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="Scope" />
+      <pane id="Scratches" />
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="manhua" type="b2602c69:ProjectViewProjectNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="manhua" type="b2602c69:ProjectViewProjectNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="manhua" type="b2602c69:ProjectViewProjectNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+              <item name="spiders" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="ShelveChangesManager" show_recycled="false">
+    <option name="remove_strategy" value="false" />
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="ecf921c8-b00f-4c0f-a340-47b61c34fcea" name="Default" comment="" />
+      <created>1555042076428</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1555042076428</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="-8" y="-8" width="1616" height="876" extended-state="6" />
+    <layout>
+      <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
+      <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
+      <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
+      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.15295629" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Docker" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="SciView" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
+      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
+      <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
+      <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
+      <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+  <component name="VcsContentAnnotationSettings">
+    <option name="myLimit" value="2678400000" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager />
+    <watches-manager />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1161">
+          <caret line="43" column="17" lean-forward="false" selection-start-line="41" selection-start-column="8" selection-end-line="43" selection-end-column="17" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="540">
+          <caret line="20" column="23" lean-forward="true" selection-start-line="20" selection-start-column="23" selection-end-line="20" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1161">
+          <caret line="43" column="35" lean-forward="false" selection-start-line="43" selection-start-column="25" selection-end-line="43" selection-end-column="35" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="0" lean-forward="true" selection-start-line="66" selection-start-column="0" selection-end-line="66" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1107">
+          <caret line="43" column="17" lean-forward="false" selection-start-line="41" selection-start-column="8" selection-end-line="43" selection-end-column="17" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1566">
+          <caret line="58" column="28" lean-forward="true" selection-start-line="58" selection-start-column="28" selection-end-line="58" selection-end-column="28" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1107">
+          <caret line="43" column="17" lean-forward="false" selection-start-line="41" selection-start-column="8" selection-end-line="43" selection-end-column="17" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1026">
+          <caret line="38" column="37" lean-forward="true" selection-start-line="38" selection-start-column="37" selection-end-line="38" selection-end-column="37" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="999">
+          <caret line="39" column="26" lean-forward="false" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="999">
+          <caret line="39" column="26" lean-forward="false" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="972">
+          <caret line="36" column="16" lean-forward="false" selection-start-line="36" selection-start-column="16" selection-end-line="36" selection-end-column="16" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="972">
+          <caret line="54" column="37" lean-forward="true" selection-start-line="54" selection-start-column="37" selection-end-line="54" selection-end-column="37" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/verydm.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="702">
+          <caret line="26" column="33" lean-forward="false" selection-start-line="26" selection-start-column="33" selection-end-line="26" selection-end-column="33" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$USER_HOME$/.PyCharm2017.3/system/python_stubs/-1627382134/nt.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="238">
+          <caret line="627" column="4" lean-forward="false" selection-start-line="627" selection-start-column="4" selection-end-line="627" selection-end-column="4" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="23" lean-forward="false" selection-start-line="66" selection-start-column="23" selection-end-line="66" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/spiders/kukukkk.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="27">
+          <caret line="20" column="23" lean-forward="false" selection-start-line="20" selection-start-column="23" selection-end-line="20" selection-end-column="23" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/manhua/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1161">
+          <caret line="43" column="17" lean-forward="false" selection-start-line="41" selection-start-column="8" selection-end-line="43" selection-end-column="17" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>

manhua/.idea/manhua.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

manhua/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (gupiao3)" project-jdk-type="Python SDK" />
+</project>

manhua/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/manhua.iml" filepath="$PROJECT_DIR$/.idea/manhua.iml" />
+    </modules>
+  </component>
+</project>

manhua/.idea/workspace.xml

@@ -0,0 +1,618 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="513c80c6-3be5-4079-b149-944cb5ce8626" name="Default" comment="" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="TRACKING_ENABLED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="CoverageDataManager">
+    <SUITE FILE_PATH="coverage/manhua$tianshangtianxia.coverage" NAME="tianshangtianxia Coverage Results" MODIFIED="1554772854431" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/spiders" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+      <file leaf-file-name="verydm.py" pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/spiders/verydm.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="243">
+              <caret line="9" column="0" lean-forward="false" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" />
+              <folding>
+                <element signature="e#24#37#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="middlewares.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/middlewares.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="296">
+              <caret line="11" column="0" lean-forward="false" selection-start-line="11" selection-start-column="0" selection-end-line="11" selection-end-column="11" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="settings.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/settings.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="-2171">
+              <caret line="66" column="24" lean-forward="false" selection-start-line="66" selection-start-column="24" selection-end-line="66" selection-end-column="24" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="items.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/items.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="270">
+              <caret line="10" column="30" lean-forward="false" selection-start-line="10" selection-start-column="30" selection-end-line="10" selection-end-column="30" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="pipelines.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/pipelines.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="529">
+              <caret line="46" column="0" lean-forward="false" selection-start-line="46" selection-start-column="0" selection-end-line="46" selection-end-column="0" />
+              <folding>
+                <element signature="e#195#206#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>SPIDER_MIDDLEWARES</find>
+      <find>DOWNLOAD_DELAY</find>
+      <find>AUTOTHROTTLE_MAX_DELAY</find>
+      <find>https://manhua.fzdm.com/15/97/</find>
+    </findStrings>
+    <replaceStrings>
+      <replace>https://manhua.fzdm.com/15/98/</replace>
+    </replaceStrings>
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/items.py" />
+        <option value="$PROJECT_DIR$/settings.py" />
+        <option value="$PROJECT_DIR$/spiders/tianshangtianxia.py" />
+        <option value="$PROJECT_DIR$/middlewares.py" />
+        <option value="$PROJECT_DIR$/pipelines.py" />
+        <option value="$PROJECT_DIR$/spiders/verydm.py" />
+      </list>
+    </option>
+  </component>
+  <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsGulpfileManager">
+    <detection-done>true</detection-done>
+    <sorting>DEFINITION_ORDER</sorting>
+  </component>
+  <component name="ProjectFrameBounds" extendedState="6">
+    <option name="x" value="-8" />
+    <option name="y" value="-8" />
+    <option name="width" value="1382" />
+    <option name="height" value="744" />
+  </component>
+  <component name="ProjectView">
+    <navigator currentView="ProjectPane" proportions="" version="1">
+      <flattenPackages />
+      <showMembers />
+      <showModules />
+      <showLibraryContents />
+      <hideEmptyPackages />
+      <abbreviatePackageNames />
+      <autoscrollToSource />
+      <autoscrollFromSource />
+      <sortByType />
+      <manualOrder />
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="Scratches" />
+      <pane id="Scope" />
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="manhua" type="b2602c69:ProjectViewProjectNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="manhua" type="b2602c69:ProjectViewProjectNode" />
+              <item name="manhua" type="462c0819:PsiDirectoryNode" />
+              <item name="spiders" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$USER_HOME$/Application Data" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="RunManager">
+    <configuration name="tianshangtianxia" type="PythonConfigurationType" factoryName="Python" temporary="true">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/spiders" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="manhua" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/spiders/tianshangtianxia.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+    </configuration>
+    <recent_temporary>
+      <list size="1">
+        <item index="0" class="java.lang.String" itemvalue="Python.tianshangtianxia" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="ShelveChangesManager" show_recycled="false">
+    <option name="remove_strategy" value="false" />
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="513c80c6-3be5-4079-b149-944cb5ce8626" name="Default" comment="" />
+      <created>1554771418691</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1554771418691</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="-8" y="-8" width="1616" height="876" extended-state="6" />
+    <editor active="true" />
+    <layout>
+      <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
+      <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
+      <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32928476" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
+      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1433162" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Docker" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="SciView" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
+      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.3994602" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
+      <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
+      <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
+      <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+  <component name="VcsContentAnnotationSettings">
+    <option name="myLimit" value="2678400000" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager />
+    <watches-manager />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1080">
+          <caret line="40" column="60" lean-forward="false" selection-start-line="40" selection-start-column="60" selection-end-line="40" selection-end-column="60" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="459">
+          <caret line="17" column="55" lean-forward="false" selection-start-line="17" selection-start-column="55" selection-end-line="17" selection-end-column="55" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="24" lean-forward="false" selection-start-line="66" selection-start-column="24" selection-end-line="66" selection-end-column="24" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="540">
+          <caret line="24" column="25" lean-forward="false" selection-start-line="24" selection-start-column="25" selection-end-line="24" selection-end-column="25" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="4212">
+          <caret line="156" column="28" lean-forward="false" selection-start-line="156" selection-start-column="28" selection-end-line="156" selection-end-column="28" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="540">
+          <caret line="24" column="25" lean-forward="false" selection-start-line="24" selection-start-column="25" selection-end-line="24" selection-end-column="25" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="4212">
+          <caret line="156" column="28" lean-forward="false" selection-start-line="156" selection-start-column="28" selection-end-line="156" selection-end-column="28" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="540">
+          <caret line="24" column="25" lean-forward="true" selection-start-line="24" selection-start-column="25" selection-end-line="24" selection-end-column="25" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="4212">
+          <caret line="156" column="28" lean-forward="true" selection-start-line="156" selection-start-column="28" selection-end-line="156" selection-end-column="28" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="351">
+          <caret line="13" column="0" lean-forward="false" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1998">
+          <caret line="74" column="33" lean-forward="true" selection-start-line="74" selection-start-column="33" selection-end-line="74" selection-end-column="33" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="243">
+          <caret line="13" column="0" lean-forward="true" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1296">
+          <caret line="48" column="45" lean-forward="false" selection-start-line="48" selection-start-column="45" selection-end-line="48" selection-end-column="45" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="999">
+          <caret line="39" column="26" lean-forward="false" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="22" lean-forward="false" selection-start-line="66" selection-start-column="22" selection-end-line="66" selection-end-column="22" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="999">
+          <caret line="39" column="26" lean-forward="false" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1782">
+          <caret line="66" column="22" lean-forward="false" selection-start-line="66" selection-start-column="22" selection-end-line="66" selection-end-column="22" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="999">
+          <caret line="39" column="26" lean-forward="false" selection-start-line="39" selection-start-column="26" selection-end-line="39" selection-end-column="26" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="432">
+          <caret line="16" column="8" lean-forward="false" selection-start-line="16" selection-start-column="8" selection-end-line="16" selection-end-column="8" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../../../gupiao3/venv2/Lib/site-packages/scrapy/spiders/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="238">
+          <caret line="88" column="8" lean-forward="false" selection-start-line="88" selection-start-column="8" selection-end-line="88" selection-end-column="8" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/requests.seen">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-3956">
+          <caret line="125" column="0" lean-forward="true" selection-start-line="125" selection-start-column="0" selection-end-line="125" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../../../gupiao3/venv2/Lib/site-packages/scrapy/downloadermiddlewares/retry.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-1145">
+          <caret line="29" column="6" lean-forward="false" selection-start-line="29" selection-start-column="6" selection-end-line="29" selection-end-column="6" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../../../gupiao3/venv2/Lib/site-packages/scrapy/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="405">
+          <caret line="15" column="10" lean-forward="true" selection-start-line="15" selection-start-column="10" selection-end-line="15" selection-end-column="10" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="259">
+          <caret line="59" column="31" lean-forward="false" selection-start-line="59" selection-start-column="31" selection-end-line="59" selection-end-column="31" />
+          <folding>
+            <element signature="e#24#37#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/items.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="270">
+          <caret line="10" column="30" lean-forward="false" selection-start-line="10" selection-start-column="30" selection-end-line="10" selection-end-column="30" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/pipelines.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="529">
+          <caret line="46" column="0" lean-forward="false" selection-start-line="46" selection-start-column="0" selection-end-line="46" selection-end-column="0" />
+          <folding>
+            <element signature="e#195#206#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/settings.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-2171">
+          <caret line="66" column="24" lean-forward="false" selection-start-line="66" selection-start-column="24" selection-end-line="66" selection-end-column="24" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/middlewares.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="296">
+          <caret line="11" column="0" lean-forward="false" selection-start-line="11" selection-start-column="0" selection-end-line="11" selection-end-column="11" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../../../manhua/venv/tianshangtianxia.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="81">
+          <caret line="3" column="9" lean-forward="true" selection-start-line="3" selection-start-column="9" selection-end-line="3" selection-end-column="9" />
+          <folding>
+            <element signature="e#0#15#0" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/spiders/verydm.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="243">
+          <caret line="9" column="0" lean-forward="false" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" />
+          <folding>
+            <element signature="e#24#37#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>

manhua/items.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class mmonlyItem(scrapy.Item):
+    siteURL = scrapy.Field()
+    detailURL = scrapy.Field()
+    title = scrapy.Field()
+    fileName = scrapy.Field()
+    path = scrapy.Field()
+    pass
+
+class ManhuaItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass
+
+
+
+
+
+

manhua/middlewares.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+import scrapy
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import time
+
+
+class AreaSpiderMiddleware(object):
+    def process_request(self, request, spider):
+        chrome_options = Options()
+        chrome_options.add_argument('--headless')  # 使用无头谷歌浏览器模式
+        chrome_options.add_argument('--disable-gpu')
+        chrome_options.add_argument('--no-sandbox')
+        # 指定谷歌浏览器路径
+        self.driver = webdriver.Chrome(chrome_options=chrome_options,executable_path='/root/zx/spider/driver/chromedriver')
+        if request.url != 'https://www.aqistudy.cn/historydata/':
+            self.driver.get(request.url)
+            time.sleep(1)
+            html = self.driver.page_source
+            self.driver.quit()
+            return scrapy.http.HtmlResponse(url=request.url, body=html.encode('utf-8'), encoding='utf-8',
+                                            request=request)
+
+class ManhuaSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class ManhuaDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+
+
+
+
+
+
+

manhua/pipelines.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+import json
+import requests
+import datetime
+
+class AirHistoryPipeline(object):
+    def open_spider(self, spider):
+        self.file = open('area.json', 'w')
+
+    def process_item(self, item, spider):
+        context = json.dumps(dict(item),ensure_ascii=False) + '\n'
+        self.file.write(context)
+        return item
+
+    def close_spider(self,spider):
+        self.file.close()
+
+class ManhuaPipeline(object):
+    def process_item(self, item, spider):
+        return item
+
+
+
+
+#
+class mmonlyPipeline(object):
+    def process_item(self, item, spider):
+        count = 0
+        detailURL = item['detailURL']
+        fileName = item['fileName']
+        # print('5555555',detailURL)
+        # print( fileName)
+        image = requests.get(detailURL)
+        # print('777777',image)
+        f= open(fileName, 'wb')
+        f.write(image.content)
+        f.close()
+        return item
+
+

manhua/requests.seen

@@ -0,0 +1,146 @@
+a3d3faa26a1025f3472a36d9b2973241d2c97b9c
+266bb69fb1f0a3fbb2395c5db96e7233e366827f
+e8bc47dbe4522d17b4d2c853717a9a8a0743b46c
+30c4d08f2165b1147597d6594b8c37e30ef4823d
+92094645ab1de2f3ce9a5a0577d0c0e02af48fd1
+da6dd1497f2dbe4578623177235f9e4a73885323
+85bd53b1b4db0b7a7f6be573a8b70027ae57e1fe
+a9f8ea2ab4599135535b34c516012b09c005c893
+76565e6758301edec502d9863728781ff8a19b50
+4984ab5cfc0e3de3ff7e0847b9c35146b3c68a2c
+7d5ea782b9ba0c78a591a73c47ebabc428a33d88
+c4fd85771a5bc3e39345b8d59172acaf366b6280
+8ba49bd8f1a92b045f03fda52f2c037503e0d601
+13977e0029aa414d7e4ee244890a5b75943ac0cb
+288b06ddc318848420be835d950cd96dc609fb37
+3abe8190cd91b3c10ac4102df491f09ae98d50c8
+ffeecd89a0a015a999203cef0ab89b86d1d2c1f2
+05e7856adbaaad325b44b4ea8bcb5576a2ffca72
+a3dcdbbf9496e8863e990ba34324b1a688d6789d
+34a54e9af60b843f291c08b80869ee62aa91a1b2
+4cafabc796848effb31955c35e779dde54b7aa03
+cc23cac031768f804f0736a7df47be89e4967116
+c6ea8fa51629ae0a213309df9f9c286cce80c11e
+38b439d2fdf76debcb9b1d32f8722746e7ce1ce8
+89ca9c5c0802f033081526d2a574611950b57b5b
+5bd3bef332b0bf38d257953f3c351a5dbb7550be
+09fe7793c653b7decafe5681edd11f67d1e20d55
+2304076a0adf57de32cc908d5385019fc830a08a
+78690bfa01210f2917f015f099fbdb56d8ebf19d
+72ad2c44fd2d7904df6a990426d5db49ebac0efd
+5ac839d659bd045216f6bcbb578a7c7a89fcf552
+fe3d5fb46a0b9bd80f1add5c8080fb92c88e0c15
+d12dffd0a97013627db84d2ee794244eef3b0c01
+73b1bb6ae830b7e93c92be9ebf5b08e262c17534
+1886d012a4f910892f0e9cb35b7100974aba5a1f
+7f831d90add4a486a7ab27329030e78c04d00d14
+eba0f59127554362b104101ce417943f1a779400
+0b0ecd4fce36eb74e9172c2f793681e5bd28ebf8
+92aac99077c7f1a07aa4c60c004f7515284095e9
+88738f9428186cee2c902744e1e934d1e0d14734
+c7f1334d5ddf60ad1d8f4d179c3bdc25f2b41625
+022e39847f35f173f60c9ed256defd589512cd48
+6d2a3d5bda39f83845f784b14bf75419e24a7bd6
+a51e46230b92b95cfe01c795dab69f505ef89e75
+da21a59d2bb364eb8bc95d7a60306e3a3998213a
+692c13f0932ddb400e565ad3019cbf06f40a82ca
+de3139747cdb2d7c75669992f2e5900d48b2b89c
+76916f92770807b8ce96b33a9733b5dce1cd34fc
+bc7efce49fd0ecd6e0f1ff5b8a3fc21b8536abad
+b07c02f56cebae5b1fe6a396478f0c78431f88b3
+bbd65e667a136d057e51ca0dbd2953e0f5f2c680
+152a0dfc6228871941a763edd1b2740c7fe32f77
+09897d891c2a0db3e80dbf02a4d4e5cfc8228ed0
+18e08e06ea2247bb81efdd094d4f6779d80f8573
+0730daaeec9fb5315827790981d71003e798575f
+9821588f9ab1ef546ce88f91dface5b0427f5be9
+af085d95e38fac416a022eb011e4a72dc78bce37
+2c7ce5601874ca38f8be0d7a387c0a202e6ac550
+01722367162912179680bed6bd70e26d423f9d8b
+3b1bf036ecb6fd7beee18592b8be7cec428ad037
+40358393b4a4ad550831d942476d1b8c6fc530ae
+55c90dc4f0da92770c8e9ca9f878c9e5595037bf
+4e08a0b8f38e7f921ebcdcd9130cdd29bab7d053
+dffa9cd4dfd59e4841230ea5c41f4f43da1f336f
+04de6b17e9ba1409662c69c825073ed777eb64d4
+ff9744a0378ebdf08417a5dab7867462ada24994
+d9136ddc7d5c6736398a3ba3007fed58f3373be9
+d550a2cfeead9ca6f6a7330cf0b8aa5ae934543a
+fe2b5abf6325692c83655b41b9e91ae62652f329
+11d89a3d3d37b8bf31e1bbdc6d985667f518ace3
+8d051ae6a5f22a7289d5ad965ce93d67c9b7f743
+01934a182d81a9bd8b66b49f29bc6c911af2d8fe
+92a77fab54d45c441fed32f45bc718ef6c2371df
+2ba4451fffb049c13d2d1c67a05bdfc4b18d6fe0
+662089f85f07c61b938f4647e228d62d51d7892a
+f6e4f6962edabeb9e0d0da6ef2fab745409fed32
+2891195d4d16f8952a44e61a48c0d8eb2127a990
+f83e9e23ae79bdf0dbce2d68660ec1a963f0e983
+17f38d7bcac6aba16dde7399800be5c109d2fcf8
+f35d3fa3c515e7bcd5b83f06a7b429272d00232b
+b57c7cfc9056b3ae110e705d1f38be694eb7d62c
+4a50bf8e49bf902a0f3778440aa9523343146395
+cb8731ca570b5243084bb8885fec5ce1ef2d09b2
+c683fc6d90315f11d8c9a412f468afd900c63e5a
+111ef48df584a3c5aeb317808c4cc7b9589761b4
+389a6911e9fa91dddae3b893276e61e07f8afa5e
+9eeb54f96c9d4293d70765802fb750ff2acd8e57
+7ffac3f90cb0e0e1855e035720ca8bab3b36d7a9
+1f1e1bbb7f28d55dc7b67232eeb94f1bd3b7e05a
+fdee7735ca06da1a1cf6fef08290d329784e3df8
+d8ba5e50ee084bf01a0b79bc04622f310dd3fc7d
+067c53860ea3f6a85714c6c6c64afe16c575385a
+4981cf7f74f6ca40efa8c8b5b8fc6a0af49623e2
+7bb49605d0a9cfc9d03d9ec3e3f9624f89b50a0e
+f4e53adeae3caefa21fc7e3ead89e65141396a52
+44da785bb5155f3739fd8aa9c961e2f96df6164c
+3f909e314990fb2d4c91a1c3f2d1a53729e7fa99
+9056129cf2909149dad1fd0976799a9a0d5733c7
+c593393c515c5813f3e7ac4664d19ca29086a723
+c45c2dc60366e77e1acfb91444be49a0e9ca2569
+abcc7dae379ef2ba022d275ccc256f1eef2457a7
+c19fc289012cff8bca0c24773ed1a63a6897a2bf
+eee82aa75562f9c6d31fac309b981ea61d171e83
+ce4cde98efb4e2168a67c5da145d122e8ecc93a5
+474f5745a3c4c6340c96f58ad85a2508ed4030ff
+034c53f395faaf6b67c0f5b9adbce822011934a4
+bb2a98a86c2ec6bc307efe1e817524d1b08c6642
+68cc57399c4c8d724f4bbceda297207d160b648f
+7e8e64c20f7e642fb8e6d45d3d10a11f2a886c3c
+c262c818ce25440a9a0038e6a75f81e545ee4c7b
+de49ae11011ecc86c2e7c91408b4a35dc7b64d6e
+fab0cb763d03604a31cfee5549c93339bae34583
+7a7b9483a5504c9abb010fc4ffd1db29215ebb9f
+d6dfa6fa1b2346af50d180e3495379e72bccb14e
+f03d48e426853da9c09569a70f783843e82caabf
+22d7904c54597644abe7e4223c221dcac01f0da2
+2dfbf228c31af12c78a9c2d98fa5aefef11e308b
+e7735859ffd6d48c1b9739ae7ad8a0b7826fc7a9
+b20fe860c512774fefb6849956d5ee060d615410
+5ffb59784fc9e5fff8f44f149e438efc0edccf9b
+6ebd4fb91935e74cadd3d66316749a5b3fcbc805
+e5723a80562a8f8a7dc192d5869cfbc67d5a6aca
+dad5a0cb794c0ab597d73c2f8bbae58d438597e0
+dbfba1344489c4267d3b2c17a657e22f8263fcb7
+3964c9c6205f5fa07e9e3db89dcb1ba815d6e6cf
+68ed0afed5313d86af640d3e7a06db8e44794e35
+13908d4785d1e2578b85153ee6591427a6d4a20b
+a9f8703c42743980e6cf0d41c121ef66762d73f6
+c1be076e45b20a8966767777215e2d65d6cedf2a
+b0c89fe5b0ebed4c4890236c9c93f5c88ab8a2ce
+105157f72674fc89a6c92562d02867e7e69eda76
+fe8910d7ed24666d6741f703cfb3e7482b0475bb
+1021196c99bd4c23d2cbe585dfcd050937720cfb
+97262b0b1f1ee99feb21c230b5eec72a979da9e2
+403c855057b15d6edf6b45706a09ae3875c73b48
+2a783f15e5ef4e967182fdd2a88f9100165bd467
+18a5604cec78935e6e2bd9a215008507d5bc2f87
+152934308df266f6339afc71ae121a74c6219997
+795c3f36cb9bbc1f02e3b2a1873fc29059c1e1db
+738d740aa55d4d23f186562f260d6fcd9ce3969b
+0d3a07bae30360151ffe1539b51478adc95ba9e4
+069caef303a327f372c8b7bfe54e0e09e8e91068
+16498cbd1b457283ed8666c6bef762fc698c021e
+af260f5dd8a565c541fea040658f8d423e8ff808
+4825d8552097303a481b76ab7093dd54a53e3e5c
+fc9550cf50d63575f2458d3ad84912cb8130bb74

manhua/settings.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for manhua project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'manhua'
+
+SPIDER_MODULES = ['manhua.spiders']
+NEWSPIDER_MODULE = 'manhua.spiders'
+
+# USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
+#
+# DOWNLOADER_MIDDLEWARES = {
+#    'manhua.middlewares.AreaSpiderMiddleware': 543,
+# }
+#
+# ITEM_PIPELINES = {
+#    'manhua.pipelines.AirHistoryPipeline': 300,
+# }
+# 渲染服务的url
+SPLASH_URL = 'http://192.168.99.100:8050'
+
+
+# 去重过滤器
+DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
+
+# 使用Splash的Http缓存
+HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
+
+
+SPIDER_MIDDLEWARES = {
+    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
+}
+
+
+#下载器中间件
+DOWNLOADER_MIDDLEWARES = {
+    'scrapy_splash.SplashCookiesMiddleware': 723,
+    'scrapy_splash.SplashMiddleware': 725,
+    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
+}
+SPIDER_MIDDLEWARES = {
+   'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
+}
+
+# 请求头
+DEFAULT_REQUEST_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+}
+
+# 管道
+# ITEM_PIPELINES = {
+#    'news.pipelines.NewsPipeline': 300,
+# }
+
+FEED_EXPORT_ENCODING = 'utf-8'
+
+ROBOTSTXT_OBEY = False
+# 默认是16，一次可以请求的最大次数
+CONCURRENT_REQUESTS = 1
+# 下载延迟
+# DOWNLOAD_DELAY = 0.1
+COOKIES_ENABLED = False
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Encoding': 'utf-8',
+    'Accept-Language': 'zh-CN,zh;q=0.8',
+    'Cache-Control': 'max-age=0',
+    'Connection': 'keep-alive',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}
+
+ITEM_PIPELINES = {'manhua.pipelines.mmonlyPipeline': 100}
+# 日志级别
+LOG_LEVEL = 'INFO'
+LOG_FILE = '/tmp/log.txt'
+
+
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'manhua (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'manhua.middlewares.ManhuaSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'manhua.middlewares.ManhuaDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'manhua.pipelines.ManhuaPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+
+# DEPTH_LIMIT = 100
+

manhua/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

manhua/spiders/kukukkk.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import os
+from manhua.items import mmonlyItem
+from scrapy.selector import Selector,HtmlXPathSelector
+from scrapy.http import Request
+from selenium import webdriver
+from scrapy_splash import SplashRequest
+
+
+
+class KukukkkSpider(scrapy.Spider):
+    name = 'kukukkk'
+    # allowed_domains = ['kukukkk.com']
+    start_urls = ['https://m.kukukkk.com/comiclist/158/']
+    title =0
+    page =0
+    retry = 0
+
+    def parse(self,response):
+        alldownload = 0
+        hxs = Selector(response=response).xpath('//div[@id="list"]/li/a/@href').extract()
+        hxs_title = Selector(response=response).xpath('//div[@id="list"]/li/a/text()').extract()
+        if alldownload == 0:
+            for url in hxs:
+                # print(url_next)
+                url_next = 'https://m.kukukkk.com/' + url
+                self.start_urls =url_next
+
+                yield SplashRequest(url=url_next, callback=self.parse_page,meta={'url':url,'title':hxs_title[self.title]})
+                self.title = self.title + 1
+        elif alldownload == 1:
+            for url in hxs:
+                if url == '/comiclist/158/19905/1.htm':
+                    url_next = 'https://m.kukukkk.com/' + url
+                    self.start_urls = url_next
+                    yield SplashRequest(url=url_next, callback=self.parse_page,meta={'url': url, 'title': hxs_title[self.title]})
+                self.title = self.title + 1
+
+    def parse_page(self, response):
+        hxs = Selector(response=response).xpath('//ul[@class="subNav"]/li/text()').extract()
+        page_total=int(hxs[1].split('/')[1] )
+        # print(response.meta['url'])
+        end_pos = response.meta['url'].rfind('/') - 1
+        start_pos = response.meta['url'].rfind('/', 0, end_pos)
+        url_s = response.meta['url'][start_pos+1:end_pos+1]
+        # print('996', response.meta['url'])
+        # print('997', start_pos)
+        # print('998', end_pos)
+        # print('999',url_s)
+        # print('111',response.meta['title'])
+
+        for i in range(1,page_total + 1):
+            if  int(i) >= 1:
+                # print(url_s)
+                url_next = 'https://m.kukukkk.com/comiclist/158/' + url_s +'/'+ str(i)+'.htm'
+                print(url_next,i,page_total)
+                f = open('F:/tmp/record.txt')
+                exit_download = 0
+                for lines in f.readlines():
+                    # print(lines)
+                    if url_next == lines.strip():
+                        print('已下载',lines)
+                        exit_download = 1
+                if exit_download == 0:
+                    yield SplashRequest(url=url_next, callback=self.parse_detail,meta={'url': url_s, 'title': response.meta['title'],'page':i})
+                    f = open('F:/tmp/record.txt','r+')
+                    f.read()
+                    f.write(url_next)
+                    f.write('\n')
+                    print('写入日志', url_next)
+                    # f.close()
+            # else:
+            #     print(i)
+
+
+
+    def parse_detail(self, response):
+        url_s = response.meta['url']
+        print('章节',url_s)
+        title_s = response.meta['title']
+        print('标题', title_s)
+        page =  response.meta['page']
+        print('页码', page)
+        filter_flag = '/comiclist/158/'+url_s+'/\d'
+        print('过滤条件',filter_flag)
+        # hxs_img = Selector(response=response).xpath('//span/img[re:test(@src, "http://m8.1whour.com/newkuku/2018/04/14/化物语_第05话/0\d+")]/@src').extract()
+        hxs_img = Selector(response=response).xpath('//a[re:test(@href, "'+filter_flag+'")]/img/@src').extract()
+        hxs_exit = Selector(response=response).xpath('//a[re:test(@href, "/exit/exit.htm")]/img/@src').extract()
+        hxs = Selector(response=response).xpath('//a[re:test(@href, "'+filter_flag+'")]/@href').extract()
+        print('图片地址',hxs_img)
+        print('最后一页图片地址', hxs_exit)
+        print('链接地址', hxs)
+        if  hxs_img :
+            for url in hxs_img:
+                # print(url)
+                item = mmonlyItem()
+                item['siteURL'] = url
+                item['title'] = title_s
+                item['path'] = 'F:/tmp/' + item['title']
+                path = item['path']
+                # name = response.xpath('//h1/text()').extract_first()
+                if not os.path.exists(path):
+                    os.makedirs(path)
+                item['detailURL'] = url
+                item['fileName'] = item['path'] + '/' + str(page) + '.jpg'  # 拼接图片名称
+                yield item
+                self.retry = 0
+        elif hxs_exit:
+            for url in hxs_exit:
+                # print(url)
+                item = mmonlyItem()
+                item['siteURL'] = url
+                item['title'] = title_s
+                item['path'] = 'F:/tmp/' + item['title']
+                path = item['path']
+                # name = response.xpath('//h1/text()').extract_first()
+                if not os.path.exists(path):
+                    os.makedirs(path)
+                item['detailURL'] = url
+                item['fileName'] = item['path'] + '/' + str(page) + '.jpg'  # 拼接图片名称
+                yield item
+                self.retry = 0
+        else:
+            if self.retry <= 3:
+                self.retry = self.retry + 1
+                print('重新发送第'+str(self.retry)+'次','https://m.kukukkk.com/comiclist/158/' + url_s + '/' + str(page) + '.htm')
+                yield SplashRequest(url='https://m.kukukkk.com/comiclist/158/' + url_s + '/' + str(page) + '.htm',callback=self.parse_detail,meta={'url': url_s, 'title': response.meta['title'], 'page': page})
+            else:
+                f = open('F:/tmp/fail.txt', 'r+')
+                f.read()
+                f.write('https://m.kukukkk.com/comiclist/158/' + url_s +'/'+ str(page)+'.htm')
+                f.write('\n')
+                print('写入失败日志', 'https://m.kukukkk.com/comiclist/158/' + url_s +'/'+ str(page)+'.htm')
+
+
+                # yield SplashRequest(url=next_urls, callback=self.parse_detail)
+
+
+

manhua/spiders/tianshangtianxia.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import os
+from manhua.items import mmonlyItem
+from scrapy.selector import Selector,HtmlXPathSelector
+from scrapy.http import Request
+from selenium import webdriver
+from scrapy_splash import SplashRequest
+
+
+class TianshangtianxiaSpider(scrapy.Spider):
+    name = 'tianshangtianxia'
+    allowed_domains = ['manhua.fzdm.com']
+    base_url = ['https://manhua.fzdm.com/2/938/']
+    start_urls = ['https://manhua.fzdm.com/2/938/']
+    visit_page = set()
+    visit_img = set()
+
+
+
+
+
+    def parse(self, response):
+        hxs = Selector(response=response).xpath('//a[re:test(@href, "index_\d+")]/@href').extract()
+        # print(hxs)
+        for url in hxs:
+            # md5_url = self.md5(url)
+            if url in self.visit_page:
+                pass
+            else:
+                self.visit_page.add(url)
+                # print(url)
+                url = 'https://manhua.fzdm.com/2/938/' + url
+                yield SplashRequest(url=url, callback=self.parse )
+
+        for url_detail in self.visit_page:
+            if url_detail in self.visit_img:
+                pass
+            else:
+                self.visit_img.add(url_detail)
+                url_detail = 'https://manhua.fzdm.com/2/938/' + url_detail
+                # print(url_detail)
+                yield SplashRequest(url=url_detail, callback=self.parse_detail,dont_filter=True)
+
+
+
+
+
+    # def md5(self, url):
+    #     import hashlib
+    #     obj = hashlib.md5()
+    #     obj.update(bytes(url, encoding='utf-8'))
+    #     return obj.hexdigest()
+    #
+    def parse_detail(self, response):
+        # print(response.text)
+        hxs = Selector(response=response).xpath('//img[@id="mhpic"]/@src').extract()
+        for url in hxs:
+            print(url)
+            item = mmonlyItem()
+            item['siteURL'] = url
+            item['title'] = response.xpath('//img[@id="mhpic"]/@alt').extract_first()
+            item['path'] = 'F:/tmp/' + item['title']
+            path = item['path']
+            name  = response.xpath('//h1/text()').extract_first()
+            if not os.path.exists(path):
+                os.makedirs(path)
+            item['detailURL'] = url
+            item['fileName'] = item['path'] + '/' + name + '.jpg'  # 拼接图片名称
+            yield item

manhua/spiders/verydm.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import os
+from manhua.items import mmonlyItem
+from scrapy.selector import Selector,HtmlXPathSelector
+from scrapy.http import Request
+from selenium import webdriver
+from scrapy_splash import SplashRequest
+import time
+
+class VerydmSpider(scrapy.Spider):
+    name = 'verydm'
+    # allowed_domains = ['verdm.com']
+    base_urls = ['http://www.verydm.com/chapter.php?id=89596']
+    start_urls = ['http://www.verydm.com/chapter.php?id=89596']
+    page =0
+
+    def parse(self, response):
+        hxs = Selector(response=response).xpath('//img[@id="mainImage2"]/@src').extract_first()
+        hxs_next = Selector(response=response).xpath('//select/option/@value').extract()
+
+        for i in hxs_next:
+            s = i.zfill(3)
+            url = 'http://imgn1.magentozh.com:8090//h/hb08tnd/ch_1/0'+s+'.jpg'
+            print(url)
+            item = mmonlyItem()
+            item['siteURL'] = url
+            # print('111111',item['siteURL'])
+            item['title'] = response.xpath('//a[re:test(@href, "http://www.verydm.com/manhua/huawuyu")]/text()').extract_first()
+            # print('222222', item['title'])
+            name = response.xpath('//div[@class="breadcrumbs"]/span/text()').extract_first()
+            # print('333333', name)
+            item['path'] = 'F:/tmp/' + item['title'] + '/' + name
+            path = item['path']
+            # print('333333', path)
+            if not os.path.exists(path):
+                os.makedirs(path)
+            item['detailURL'] = url
+            item['fileName'] = item['path'] + '/' + name + '/' + str(i)  +'.jpg'  # 拼接图片名称
+            yield item
+            time.sleep(10)
+

scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = manhua.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = manhua