Roman/HackingScripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

only parse html files

Browse Source
This commit is contained in:
Roman Hergenreder 2022-01-24 20:01:49 +01:00
parent 8adc30d7ae
commit 631dade746
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
crawl_urls.py Normal file → Executable file
View File

@ -1,3 +1,5 @@
#!/usr/bin/env python3
import argparse import argparse
import urllib.parse import urllib.parse
import urllib3 import urllib3
@ -55,6 +57,10 @@ class Crawler:
self.visited.add(url) self.visited.add(url)
res = self.request(url) res = self.request(url)
content_type = res.headers.get("Content-Type", None)
if "text/html" not in content_type.lower().split(";"):
continue
urls = self.collect_urls(res.text) urls = self.collect_urls(res.text)
for url in urls: for url in urls:
parts = urllib.parse.urlparse(url) parts = urllib.parse.urlparse(url)