Teşekkürler işime yaradı tam da sitemap lazımken. PHP ile yapıyordum bu da işe yarar.

Hariç tutulacak kelime listesi ve 1 saniye bekleme ekledim.

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import pyfiglet
import time

ascii_banner = pyfiglet.figlet_format("Gelistir")
print(ascii_banner)

IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp')
WEBPAGE_EXTENSIONS = ('.html', '.htm', '.php', '.asp', '.aspx', '.jsp', '.jspx', '.cgi', '.pl')

# Hariç tutulacak kelimeleri içeren liste
EXCLUDED_WORDS = ['admin', 'login', 'logout', 'register', 'forgot', 'private', 'cart', 'protection', '+tel', 'javascript', 'mailto:', 'tel:', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'jpg', 'jpeg', 'png', 'gif', 'bmp', 'svg', 'webp', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'jpg', 'jpeg', 'png', 'gif', 'bmp', 'svg', 'webp', 'youtube', 'facebook', 'twitter', 'instagram', 'linkedin', 'pinterest', 'tumblr', 'snapchat', 'whatsapp', 'viber', 'telegram', 'skype', 'discord', 'github', 'gitlab', 'bitbucket', 'stackoverflow', 'wikipedia', 'google', 'yahoo', 'bing', 'yandex', 'duckduckgo', 'baidu', 'ask', 'aol', 'msn', 'mail', 'outlook', 'hotmail', 'gmail', 'yahoo', 'yandex', 'aol', 'protonmail', 'tutanota', 'zoho', 'icloud', 'inbox', 'mailfence', 'gmx', 'mail.com', 'yandex.com']

def get_links(url):
try:
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = [a.get('href') for a in soup.find_all('a', href=True)]
absolute_links = [urljoin(url, link) for link in links]
return absolute_links
except Exception as e:
print(f"Hata: {str(e)}")
return []

def is_valid_url(url):
parsed_url = urlparse(url)
return (parsed_url.path.endswith(WEBPAGE_EXTENSIONS) or not parsed_url.path or '.' not in parsed_url.path.split('/')[-1]) and not parsed_url.path.endswith(IMAGE_EXTENSIONS)

def contains_excluded_word(url):
return any(word in url.lower() for word in EXCLUDED_WORDS)

def crawl_site(root_url, domain, max_depth=-1):
visited = set()
queue = [(root_url, 0)]

while queue:
url, depth = queue.pop(0)
if max_depth != -1 and depth > max_depth:
continue
        if url in visited:
continue
        visited.add(url)

if domain not in url:
continue

        if not is_valid_url(url) or contains_excluded_word(url):
continue

        print(f"Processing: {url}")

links = get_links(url)
for link in links:
if link not in visited and domain in link:
queue.append((link, depth + 1))

# 1 saniye bekle
        time.sleep(1)

sitemap_filename = input("Sitemap dosyasının adını girin (örneğin, sitemap.xml): ")
create_sitemap(root_url, visited, sitemap_filename)

def create_sitemap(start_url, visited, filename):
with open(filename, 'w', encoding='utf-8') as file:
file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
file.write('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n')
for url in visited:
file.write(f'<url><loc>{url}</loc></url>\n')
file.write('</urlset>\n')

if __name__ == "__main__":
root_url = input("Web sitesi URL'sini girin: ")
domain = input("Dahil etmek istediğiniz alan adını girin (örneğin, smsvar.com): ")
max_depth = int(input("Derinlik sınırlamasını girin (tam tarama için -1): "))

    crawl_site(root_url, domain, max_depth)