Stabilized v1.2.3

Author: OSINT-TECHNOLOGIES

README.md

@@ -78,6 +78,11 @@ DPULSE is a software solution for conducting OSINT research in relation to a cer
     - SecurityTrails API (deep subdomains and DNS enumeration)
     - HudsonRock API (for querying a database with exposed computers which were compromised through global info-stealer campaigns)
 
+5. ***Web-pages snapshoting:*** extended functionality which allows to save web-pages copies in different forms:
+    - Screenshot snapshotting (saves target domain's page in form of screenshot)
+    - HTML snapshotting (saves target domain'spage in form of HTML file)
+    - Wayback Machine snapshotting (saves every version of target domain's page within a user-defined time period)
+
 Finally, DPULSE compiles all found data into an easy-to-read HTML or XLSX report by category. It also saves all information about scan in local report storage database, which can be restored later.
 
 # How to install and run DPULSE
@@ -166,7 +171,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
 # Tasks to complete before new release
 - [x] Add web pages snapshoting (with screenshots)
 - [x] Add web pages snapshoting (with web pages copying as HTML objects)
-- [ ] Add web pages snapshoting (with Wayback Machine)
+- [x] Add web pages snapshoting (with Wayback Machine)
 
 # DPULSE mentions in social medias
 
@@ -176,6 +181,8 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
 
 ### [The very first mention from cybercrime intelligence company (HudsonRock)](https://www.linkedin.com/feed/update/urn:li:share:7294336938495385600/)
 
+### [The very first mention on cybersecurity educational website (Ethical Hackers Academy)](https://ethicalhacksacademy.com/blogs/cyber-security-tools/dpulse)
+
 ## X.com mentions:
 
 ### [by @DarkWebInformer](https://x.com/DarkWebInformer/status/1787583156775759915?t=Ak1W9ddUPpDvLAkVyQG8fQ&s=19)

datagather_modules/data_assembler.py

@@ -16,6 +16,7 @@
 from screen_snapshotting import take_screenshot
 from config_processing import read_config
 from html_snapshotting import save_page_as_html
+from archive_snapshotting import download_snapshot
 
 try:
     import requests
@@ -72,7 +73,7 @@ def report_preprocessing(self, short_domain, report_file_type):
         os.makedirs(report_folder, exist_ok=True)
         return casename, db_casename, db_creation_date, robots_filepath, sitemap_filepath, sitemap_links_filepath, report_file_type, report_folder, files_ctime, report_ctime
 
-    def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, keywords, keywords_flag, dorking_flag, used_api_flag, snapshotting_flag, username):
+    def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, keywords, keywords_flag, dorking_flag, used_api_flag, snapshotting_flag, username, from_date, end_date):
         casename, db_casename, db_creation_date, robots_filepath, sitemap_filepath, sitemap_links_filepath, report_file_type, report_folder, ctime, report_ctime = self.report_preprocessing(short_domain, report_file_type)
         logging.info(f'### THIS LOG PART FOR {casename} CASE, TIME: {ctime} STARTS HERE')
         print(Fore.GREEN + "Started scanning domain" + Style.RESET_ALL)
@@ -186,6 +187,8 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                     take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
                 elif snapshotting_flag.lower() == 'p':
                     save_page_as_html(url, report_folder + '//domain_html_copy.html')
+                elif snapshotting_flag.lower() == 'w':
+                    download_snapshot(short_domain, from_date, end_date, report_folder)
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
             else:
                 pass
@@ -269,6 +272,8 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                     take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
                 elif snapshotting_flag.lower() == 'p':
                     save_page_as_html(url, report_folder + '//domain_html_copy.html')
+                elif snapshotting_flag.lower() == 'w':
+                    download_snapshot(short_domain, from_date, end_date, report_folder)
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
             else:
                 pass

dpulse.py

@@ -53,17 +53,17 @@
 cli = cli_init.Menu()
 cli.welcome_menu()
 
-def process_report(report_filetype, short_domain, url, case_comment, keywords_list, keywords_flag, dorking_flag, used_api_flag, pagesearch_flag, pagesearch_ui_mark, spinner_thread, snapshotting_flag, snapshotting_ui_mark, username):
+def process_report(report_filetype, short_domain, url, case_comment, keywords_list, keywords_flag, dorking_flag, used_api_flag, pagesearch_flag, pagesearch_ui_mark, spinner_thread, snapshotting_flag, snapshotting_ui_mark, username, from_date, end_date):
     import xlsx_report_creation as xlsx_rc
     import html_report_creation as html_rc
     from misc import time_processing
 
     try:
         start = time()
         if pagesearch_flag in ['y', 'si']:
-            data_array, report_info_array = data_processing.data_gathering(short_domain, url, report_filetype.lower(), pagesearch_flag.lower(), keywords_list, keywords_flag, dorking_flag.lower(), used_api_flag, snapshotting_flag, username)
+            data_array, report_info_array = data_processing.data_gathering(short_domain, url, report_filetype.lower(), pagesearch_flag.lower(), keywords_list, keywords_flag, dorking_flag.lower(), used_api_flag, snapshotting_flag, username, from_date, end_date)
         else:
-            data_array, report_info_array = data_processing.data_gathering(short_domain, url, report_filetype.lower(), pagesearch_flag.lower(), '', keywords_flag, dorking_flag.lower(), used_api_flag, snapshotting_flag, username)
+            data_array, report_info_array = data_processing.data_gathering(short_domain, url, report_filetype.lower(), pagesearch_flag.lower(), '', keywords_flag, dorking_flag.lower(), used_api_flag, snapshotting_flag, username, from_date, end_date)
         end = time() - start
         endtime_string = time_processing(end)
 
@@ -164,7 +164,7 @@ def run():
                                     else:
                                         print(Fore.RED + "\nInvalid API usage mode" + Style.RESET_ALL)
                                         break
-                                    snapshotting_flag = input(Fore.YELLOW + "Select Snapshotting mode [S(creenshot)/P(age Copy)/N (for None)] >> ")
+                                    snapshotting_flag = input(Fore.YELLOW + "Select Snapshotting mode [S(creenshot)/P(age Copy)/W(ayback Machine)/N (for None)] >> ")
                                     if pagesearch_flag.lower() == 'y' or pagesearch_flag.lower() == 'n':
                                         if pagesearch_flag.lower() == "n":
                                             pagesearch_ui_mark = 'No'
@@ -196,11 +196,16 @@ def run():
                                             break
                                         else:
                                             snapshotting_ui_mark = 'No'
+                                            from_date = end_date = 'N'
                                             if snapshotting_flag.lower() == 's':
+                                                from_date = end_date = 'N'
                                                 snapshotting_ui_mark = "Yes, domain's main page snapshotting as a screenshot"
                                             elif snapshotting_flag.lower() == 'p':
+                                                from_date = end_date = 'N'
                                                 snapshotting_ui_mark = "Yes, domain's main page snapshotting as a .HTML file"
                                             elif snapshotting_flag.lower() == 'w': # not supported at the moment
+                                                from_date = str(input('Enter start date (YYYYMMDD format): '))
+                                                end_date = str(input('Enter end date (YYYYMMDD format): '))
                                                 snapshotting_ui_mark = "Yes, domain's main page snapshotting using Wayback Machine"
                                         cli_init.print_prescan_summary(short_domain, report_filetype.upper(), pagesearch_ui_mark, dorking_ui_mark, used_api_ui, case_comment, snapshotting_ui_mark)
                                         print(Fore.LIGHTMAGENTA_EX + "[BASIC SCAN START]\n" + Style.RESET_ALL)
@@ -209,7 +214,7 @@ def run():
                                         if report_filetype.lower() in ['html', 'xlsx']:
                                             process_report(report_filetype, short_domain, url, case_comment,
                                                            keywords_list, keywords_flag, dorking_flag, used_api_flag,
-                                                           pagesearch_flag, pagesearch_ui_mark, spinner_thread, snapshotting_flag, snapshotting_ui_mark, username)
+                                                           pagesearch_flag, pagesearch_ui_mark, spinner_thread, snapshotting_flag, snapshotting_ui_mark, username, from_date, end_date)
                                     else:
                                         print(Fore.RED + "\nUnsupported PageSearch mode. Please choose between Y or N")
 

service/cli_init.py

@@ -20,7 +20,7 @@ def welcome_menu(self):
         fig = Figlet(font=wm_font)
         print('\n')
         self.console.print(fig.renderText('DPULSE'), style=preview_style)
-        print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2.2 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL)
+        print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2.3 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL)
         print(Fore.MAGENTA + Style.BRIGHT + '[Visit our pages]\nGitHub repository: https://github.com/OSINT-TECHNOLOGIES\nPyPi page: https://pypi.org/project/dpulse/\nDocumentation: https://dpulse.readthedocs.io' + Style.RESET_ALL)
 
     def print_main_menu(self):

service/config_processing.py

@@ -27,10 +27,11 @@ def create_config():
     ]
 
     config = configparser.ConfigParser()
+    config['HTML_REPORTING'] = {'template': 'default'}
     config['LOGGING'] = {'log_level': 'info'}
     config['CLI VISUAL'] = {'preview_color': 'red', 'font': 'slant'}
     config['DORKING'] = {'dorking_delay (secs)': '2', 'delay_step': '5'}
-    config['SNAPSHOTTING'] = {'installed_browser': 'firefox', 'opera_browser_path': 'None'}
+    config['SNAPSHOTTING'] = {'installed_browser': 'firefox', 'opera_browser_path': 'None', 'wayback_retries': '3', 'wayback_req_pause': '2'}
     config['USER-AGENTS'] = {}
     for i, agent in enumerate(basic_user_agents):
         config['USER-AGENTS'][f'agent_{i + 1}'] = agent
@@ -56,6 +57,9 @@ def read_config():
     proxies_file_path = config.get('PROXIES', 'proxies_file_path')
     installed_browser = config.get('SNAPSHOTTING', 'installed_browser')
     opera_browser_path = config.get('SNAPSHOTTING', 'opera_browser_path')
+    wayback_retries_amount = config.get('SNAPSHOTTING', 'wayback_retries')
+    wayback_requests_pause = config.get('SNAPSHOTTING', 'wayback_req_pause')
+    html_report_template = config.get('HTML_REPORTING', 'template')
 
 
     config_values = {
@@ -67,7 +71,10 @@ def read_config():
         'user_agents': user_agents,
         'proxies_file_path': proxies_file_path,
         'installed_browser': installed_browser,
-        'opera_browser_path': opera_browser_path
+        'opera_browser_path': opera_browser_path,
+        'wayback_retries_amount': wayback_retries_amount,
+        'wayback_requests_pause': wayback_requests_pause,
+        'template': html_report_template
     }
 
     return config_values

service/pdf_report_templates/compromise_report_template.html

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <style>
+        body { font-family: Arial, sans-serif; line-height: 1.6; }
+        h3 { text-align: center; margin-top: 20px; }
+        pre { background: #f8f8f8; padding: 5px; border: 1px solid #ddd; }
+        .section { margin: 10px 40px; }
+    </style>
+</head>
+<body>
+
+<h3>OPEN SOURCE RESEARCH REPORT</h3>
+<p><b>Organization:</b> {{org}}</p>
+<hr />
+
+<h3>TABLE OF CONTENTS</h3>
+<div class="section">
+  <p>1. General scan information</p>
+  <p>2. WHOIS information</p>
+  <p>3. Social medias links</p>
+  <p>4. Subdomains information</p>
+  <p>5. DNS & SSL information</p>
+  <p>6. Services & frameworks</p>
+  <p>7. Basic pre-pentest information</p>
+  <p>8. Dorking scan info</p>
+  <p>9. PageSearch results</p>
+  <p>10. API scan results</p>
+</div>
+<hr />
+
+<h3>GENERAL SCAN INFO</h3>
+<div class="section">
+  <p><b>Total subdomains:</b> {{a_tsf}}</p>
+  <p><b>Social media links:</b> {{a_tsm}}</p>
+  <pre>Robots.txt: {{robots_txt_result}}
+Sitemap.xml: {{sitemap_xml_result}}
+Dorking: {{dorking_status}}</pre>
+</div>
+<hr />
+
+<h3>WHOIS INFORMATION</h3>
+<div class="section">
+  <p><b>Domain:</b> {{sh_domain}} <b>URL:</b> {{full_url}}</p>
+  <pre>IP: {{ip_address}}
+Registrar: {{registrar}}
+Dates: {{creation_date}} → {{expiration_date}}</pre>
+</div>
+<hr />
+
+<h3>SOCIAL MEDIAS</h3>
+<div class="section">
+  <p><b>Facebook:</b></p><pre>{% for l in fb_links %}⇒ {{ l }}{% endfor %}</pre>
+  <p><b>Twitter/X:</b></p><pre>{% for l in tw_links+xcom_links %}⇒ {{ l }}{% endfor %}</pre>
+  <p><b>Instagram:</b></p><pre>{% for l in inst_links %}⇒ {{ l }}{% endfor %}</pre>
+</div>
+<hr />
+
+<h3>SUBDOMAINS</h3>
+<div class="section">
+  <p><b>Found subdomains:</b></p><pre>{% for sd in subdomains %}⇒ {{ sd }}{% endfor %}</pre>
+  <p><b>IPs:</b></p><pre>{% for sdip in subdomain_ip %}⇒ {{ sdip }}{% endfor %}</pre>
+</div>
+<hr />
+
+<h3>DNS/SSL</h3>
+<div class="section">
+  <pre>NS: {{name_servers}}
+MX: {{mx_records}}
+SSL Issuer: {{issuer}}
+NotBefore: {{notBefore}}
+NotAfter: {{notAfter}}</pre>
+</div>
+<hr />
+
+<h3>SERVICES</h3>
+<div class="section">
+  <p><b>Web servers:</b></p><pre>{% for ws in web_servers %}⇒ {{ ws }}{% endfor %}</pre>
+  <p><b>CMS:</b></p><pre>{% for cm in cms %}⇒ {{ cm }}{% endfor %}</pre>
+  <p><b>Languages:</b></p><pre>{% for pl in programming_languages %}⇒ {{ pl }}{% endfor %}</pre>
+</div>
+<hr />
+
+<h3>BASIC PRE-PENTEST</h3>
+<div class="section">
+  <p><b>Open ports:</b></p><pre>{% for op in ports %}⇒ {{ op }}{% endfor %}</pre>
+  <p><b>Vulnerabilities:</b></p><pre>{% for vuln in vulns %}⇒ {{ vuln }}{% endfor %}</pre>
+</div>
+<hr />
+
+<h3>DORKING SCAN</h3>
+<div class="section"><pre>{{ add_dsi | safe }}</pre></div>
+<hr />
+
+<h3>PAGESEARCH</h3>
+<div class="section">
+  <pre>Subdomains: {{ps_s}}
+Emails: {{ps_e}}
+Documents: {{ps_f}}
+…</pre>
+</div>
+<hr />
+
+<h3>VIRUSTOTAL</h3>
+<div class="section"><pre>{{ virustotal_output }}</pre></div>
+<h3>SECURITYTRAILS</h3>
+<div class="section"><pre>{{ securitytrails_output }}</pre></div>
+<h3>HUDSONROCK</h3>
+<div class="section"><pre>{{ hudsonrock_output }}</pre></div>
+<hr />
+
+<p style="text-align:center;">Created by DPULSE (OSINT-TECHNOLOGIES)</p>
+<p style="text-align:center;">
+  <a href="https://github.com/OSINT-TECHNOLOGIES">GitHub</a> |
+  <a href="https://pypi.org/project/dpulse/">PyPI</a>
+</p>
+
+</body>
+</html>