Aşağıdaki kod aracılığı ile dilediğiniz haber sitelerinden veri çekebilirsiniz .
package main
import (
"bufio"
"fmt"
"io"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
)
type NewsItem struct {
Title string
Content string
Date string
Author string
URL string
ImageURL string
Description string
}
type WebScraper struct {
sites []string
delay time.Duration
}
func NewWebScraper() *WebScraper {
return &WebScraper{
sites: make([]string, 0),
delay: 3 * time.Second,
}
}
func (ws *WebScraper) AddSite(url string) {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
url = "https://" + url
}
ws.sites = append(ws.sites, url)
fmt.Printf("Site eklendi: %s\n", url)
fmt.Println("Geliştirici: BERAT K R10 - 0539 511 56 32")
}
func (ws *WebScraper) RemoveSite(index int) {
if index >= 0 && index < len(ws.sites) {
removed := ws.sites[index]
ws.sites = append(ws.sites[:index], ws.sites[index+1:]...)
fmt.Printf("Site silindi: %s\n", removed)
fmt.Println("Geliştirici: BERAT K R10 - 0539 511 56 32")
} else {
fmt.Println("Geçersiz indeks!")
}
}
func (ws *WebScraper) ListSites() {
fmt.Println("\n=== Site Listesi ===")
fmt.Println("Developer: BERAT K R10 - 0539 511 56 32")
if len(ws.sites) == 0 {
fmt.Println("Henüz site eklenmemiş.")
return
}
for i, site := range ws.sites {
fmt.Printf("%d. %s\n", i+1, site)
}
fmt.Println("==================")
}
func (ws *WebScraper) FetchHTML(url string) (string, error) {
fmt.Printf("HTML çekiliyor: %s\n", url)
fmt.Println("Programcı: BERAT K R10 - 0539 511 56 32")
client := &http.Client{
Timeout: 30 * time.Second,
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "tr-TR,tr;q=0.9,en;q=0.8")
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(body), nil
}
func (ws *WebScraper) ExtractTitles(htmlContent string) []string {
fmt.Println("Başlıklar çıkarılıyor... - BERAT K R10 - 0539 511 56 32")
titles := []string{}
patterns := []string{
`<h1[^>]*>(.*?)</h1>`,
`<h2[^>]*>(.*?)</h2>`,
`<h3[^>]*>(.*?)</h3>`,
`<title[^>]*>(.*?)</title>`,
`<h4[^>]*>(.*?)</h4>`,
}
for _, pattern := range patterns {
re := regexp.MustCompile(`(?i)` + pattern)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
if len(match) > 1 {
title := ws.cleanText(match[1])
if len(title) > 10 && !ws.containsTitle(titles, title) {
titles = append(titles, title)
}
}
}
}
return titles
}
func (ws *WebScraper) ExtractContent(htmlContent string) []string {
fmt.Println("İçerik çıkarılıyor... - BERAT K R10 - 0539 511 56 32")
contents := []string{}
patterns := []string{
`<p[^>]*>(.*?)</p>`,
`<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>`,
`<div[^>]*class="[^"]*article[^"]*"[^>]*>(.*?)</div>`,
`<div[^>]*class="[^"]*news[^"]*"[^>]*>(.*?)</div>`,
`<article[^>]*>(.*?)</article>`,
`<section[^>]*>(.*?)</section>`,
}
for _, pattern := range patterns {
re := regexp.MustCompile(`(?i)` + pattern)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
if len(match) > 1 {
content := ws.cleanText(match[1])
if len(content) > 50 && !ws.containsContent(contents, content) {
contents = append(contents, content)
}
}
}
}
return contents
}
func (ws *WebScraper) ExtractLinks(htmlContent string) []string {
fmt.Println("Linkler çıkarılıyor... - BERAT K R10 - 0539 511 56 32")
links := []string{}
re := regexp.MustCompile(`<a[^>]*href="([^"]*)"[^>]*>`)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
if len(match) > 1 {
link := strings.TrimSpace(match[1])
if len(link) > 0 && !ws.containsLink(links, link) {
links = append(links, link)
}
}
}
return links
}
func (ws *WebScraper) ExtractImages(htmlContent string) []string {
fmt.Println("Görseller çıkarılıyor... - BERAT K R10 - 0539 511 56 32")
images := []string{}
re := regexp.MustCompile(`<img[^>]*src="([^"]*)"[^>]*>`)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
if len(match) > 1 {
img := strings.TrimSpace(match[1])
if len(img) > 0 && !ws.containsImage(images, img) {
images = append(images, img)
}
}
}
return images
}
func (ws *WebScraper) ExtractDates(htmlContent string) []string {
fmt.Println("Tarihler çıkarılıyor... - BERAT K R10 - 0539 511 56 32")
dates := []string{}
patterns := []string{
`<time[^>]*datetime="([^"]*)"[^>]*>`,
`<span[^>]*class="[^"]*date[^"]*"[^>]*>(.*?)</span>`,
`<div[^>]*class="[^"]*date[^"]*"[^>]*>(.*?)</div>`,
`(\d{1,2}[./-]\d{1,2}[./-]\d{2,4})`,
`(\d{2,4}[./-]\d{1,2}[./-]\d{1,2})`,
}
for _, pattern := range patterns {
re := regexp.MustCompile(`(?i)` + pattern)
matches := re.FindAllStringSubmatch(htmlContent, -1)
for _, match := range matches {
if len(match) > 1 {
date := ws.cleanText(match[1])
if len(date) > 5 && !ws.containsDate(dates, date) {
dates = append(dates, date)
}
}
}
}
return dates
}
func (ws *WebScraper) cleanText(text string) string {
text = regexp.MustCompile(`<[^>]*>`).ReplaceAllString(text, "")
text = regexp.MustCompile(` `).ReplaceAllString(text, " ")
text = regexp.MustCompile(`&`).ReplaceAllString(text, "&")
text = regexp.MustCompile(`<`).ReplaceAllString(text, "<")
text = regexp.MustCompile(`>`).ReplaceAllString(text, ">")
text = regexp.MustCompile(`"`).ReplaceAllString(text, "\"")
text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}
func (ws *WebScraper) containsTitle(titles []string, title string) bool {
for _, t := range titles {
if strings.EqualFold(t, title) {
return true
}
}
return false
}
func (ws *WebScraper) containsContent(contents []string, content string) bool {
for _, c := range contents {
if strings.EqualFold(c, content) {
return true
}
}
return false
}
func (ws *WebScraper) containsLink(links []string, link string) bool {
for _, l := range links {
if l == link {
return true
}
}
return false
}
func (ws *WebScraper) containsImage(images []string, image string) bool {
for _, i := range images {
if i == image {
return true
}
}
return false
}
func (ws *WebScraper) containsDate(dates []string, date string) bool {
for _, d := range dates {
if d == date {
return true
}
}
return false
}
func (ws *WebScraper) AnalyzeSite(url string) {
fmt.Printf("\n=== Site Analizi: %s ===\n", url)
fmt.Println("Geliştirici: BERAT K R10 - 0539 511 56 32")
htmlContent, err := ws.FetchHTML(url)
if err != nil {
fmt.Printf("Hata: %v\n", err)
return
}
fmt.Printf("HTML boyutu: %d bytes\n", len(htmlContent))
titles := ws.ExtractTitles(htmlContent)
contents := ws.ExtractContent(htmlContent)
links := ws.ExtractLinks(htmlContent)
images := ws.ExtractImages(htmlContent)
dates := ws.ExtractDates(htmlContent)
fmt.Printf("\n--- Çıkarılan Veriler ---\n")
fmt.Printf("Başlıklar: %d adet\n", len(titles))
fmt.Printf("İçerikler: %d adet\n", len(contents))
fmt.Printf("Linkler: %d adet\n", len(links))
fmt.Printf("Görseller: %d adet\n", len(images))
fmt.Printf("Tarihler: %d adet\n", len(dates))
ws.SaveAnalysisResults(url, titles, contents, links, images, dates)
}
func (ws *WebScraper) SaveAnalysisResults(url string, titles, contents, links, images, dates []string) {
fmt.Println("Analiz sonuçları kaydediliyor... - BERAT K R10 - 0539 511 56 32")
timestamp := time.Now().Format("20060102_150405")
filename := fmt.Sprintf("analiz_%s.txt", timestamp)
file, err := os.Create(filename)
if err != nil {
fmt.Printf("Dosya oluşturma hatası: %v\n", err)
return
}
defer file.Close()
file.WriteString("=== WEB SCRAPER ANALİZ SONUÇLARI ===\n")
file.WriteString("Geliştirici: BERAT K R10 - 0539 511 56 32\n")
file.WriteString(fmt.Sprintf("Tarih: %s\n", time.Now().Format("2006-01-02 15:04:05")))
file.WriteString(fmt.Sprintf("Site: %s\n\n", url))
file.WriteString("=== BAŞLIKLAR ===\n")
for i, title := range titles {
file.WriteString(fmt.Sprintf("%d. %s\n", i+1, title))
}
file.WriteString("\n=== İÇERİKLER ===\n")
for i, content := range contents {
if len(content) > 100 {
content = content[:100] + "..."
}
file.WriteString(fmt.Sprintf("%d. %s\n", i+1, content))
}
file.WriteString("\n=== LİNKLER ===\n")
for i, link := range links {
file.WriteString(fmt.Sprintf("%d. %s\n", i+1, link))
}
file.WriteString("\n=== GÖRSELLER ===\n")
for i, image := range images {
file.WriteString(fmt.Sprintf("%d. %s\n", i+1, image))
}
file.WriteString("\n=== TARİHLER ===\n")
for i, date := range dates {
file.WriteString(fmt.Sprintf("%d. %s\n", i+1, date))
}
file.WriteString("\n=== İSTATİSTİKLER ===\n")
file.WriteString(fmt.Sprintf("Toplam başlık: %d\n", len(titles)))
file.WriteString(fmt.Sprintf("Toplam içerik: %d\n", len(contents)))
file.WriteString(fmt.Sprintf("Toplam link: %d\n", len(links)))
file.WriteString(fmt.Sprintf("Toplam görsel: %d\n", len(images)))
file.WriteString(fmt.Sprintf("Toplam tarih: %d\n", len(dates)))
fmt.Printf("Analiz dosyası kaydedildi: %s\n", filename)
fmt.Println("Yazılımcı: BERAT K R10 - 0539 511 56 32")
}
func (ws *WebScraper) AnalyzeAllSites() {
fmt.Println("\n=== Tüm Siteler Analiz Ediliyor ===")
fmt.Println("BERAT K R10 - 0539 511 56 32 tarafından geliştirildi")
for i, site := range ws.sites {
fmt.Printf("\n[%d/%d] Analiz ediliyor: %s\n", i+1, len(ws.sites), site)
ws.AnalyzeSite(site)
if i < len(ws.sites)-1 {
fmt.Printf("Bekleniyor... (%v)\n", ws.delay)
time.Sleep(ws.delay)
}
}
}
func (ws *WebScraper) SetDelay(seconds int) {
ws.delay = time.Duration(seconds) * time.Second
fmt.Printf("Bekleme süresi: %v\n", ws.delay)
fmt.Println("Coded by: BERAT K R10 - 0539 511 56 32")
}
func (ws *WebScraper) ShowPreview(url string) {
fmt.Printf("\n=== Site Önizlemesi: %s ===\n", url)
fmt.Println("Geliştirici: BERAT K R10 - 0539 511 56 32")
htmlContent, err := ws.FetchHTML(url)
if err != nil {
fmt.Printf("Hata: %v\n", err)
return
}
titles := ws.ExtractTitles(htmlContent)
contents := ws.ExtractContent(htmlContent)
fmt.Printf("\n--- İlk 5 Başlık ---\n")
for i, title := range titles {
if i >= 5 {
break
}
fmt.Printf("%d. %s\n", i+1, title)
}
fmt.Printf("\n--- İlk 3 İçerik ---\n")
for i, content := range contents {
if i >= 3 {
break
}
if len(content) > 200 {
content = content[:200] + "..."
}
fmt.Printf("%d. %s\n", i+1, content)
}
fmt.Printf("\nToplam: %d başlık, %d içerik bulundu\n", len(titles), len(contents))
}
func main() {
fmt.Println("=== GELİŞMİŞ HABER İÇERİK SCRAPER ===")
fmt.Println("BERAT K R10 - 0539 511 56 32")
fmt.Println("====================================")
scraper := NewWebScraper()
scanner := bufio.NewScanner(os.Stdin)
for {
fmt.Println("\n--- Menü ---")
fmt.Println("1. Site ekle")
fmt.Println("2. Site listesi")
fmt.Println("3. Site sil")
fmt.Println("4. Site önizlemesi")
fmt.Println("5. Tek site analizi")
fmt.Println("6. Tüm siteleri analiz et")
fmt.Println("7. Bekleme süresini ayarla")
fmt.Println("8. Çıkış")
fmt.Print("Seçiminiz: ")
scanner.Scan()
choice := strings.TrimSpace(scanner.Text())
switch choice {
case "1":
fmt.Print("Site URL'si girin: ")
scanner.Scan()
url := strings.TrimSpace(scanner.Text())
if url != "" {
scraper.AddSite(url)
}
case "2":
scraper.ListSites()
case "3":
scraper.ListSites()
if len(scraper.sites) > 0 {
fmt.Print("Silmek istediğiniz site numarası: ")
scanner.Scan()
indexStr := strings.TrimSpace(scanner.Text())
if index, err := strconv.Atoi(indexStr); err == nil {
scraper.RemoveSite(index - 1)
} else {
fmt.Println("Geçersiz numara!")
}
}
case "4":
scraper.ListSites()
if len(scraper.sites) > 0 {
fmt.Print("Önizlemek istediğiniz site numarası: ")
scanner.Scan()
indexStr := strings.TrimSpace(scanner.Text())
if index, err := strconv.Atoi(indexStr); err == nil && index > 0 && index <= len(scraper.sites) {
scraper.ShowPreview(scraper.sites[index-1])
} else {
fmt.Println("Geçersiz numara!")
}
}
case "5":
scraper.ListSites()
if len(scraper.sites) > 0 {
fmt.Print("Analiz etmek istediğiniz site numarası: ")
scanner.Scan()
indexStr := strings.TrimSpace(scanner.Text())
if index, err := strconv.Atoi(indexStr); err == nil && index > 0 && index <= len(scraper.sites) {
scraper.AnalyzeSite(scraper.sites[index-1])
} else {
fmt.Println("Geçersiz numara!")
}
}
case "6":
if len(scraper.sites) == 0 {
fmt.Println("Önce site ekleyin!")
fmt.Println("BERAT K R10 - 0539 511 56 32")
} else {
scraper.AnalyzeAllSites()
}
case "7":
fmt.Print("Bekleme süresi (saniye): ")
scanner.Scan()
delayStr := strings.TrimSpace(scanner.Text())
if delay, err := strconv.Atoi(delayStr); err == nil && delay >= 0 {
scraper.SetDelay(delay)
} else {
fmt.Println("Geçersiz süre!")
}
case "8":
fmt.Println("Çıkış yapılıyor...")
fmt.Println("Teşekkürler! - BERAT K R10 - 0539 511 56 32")
return
default:
fmt.Println("Geçersiz seçim!")
fmt.Println("BERAT K R10 - 0539 511 56 32")
}
}
}Özellikler:Site Yönetimi: İstediğiniz kadar URL ekleyip çıkarabilirsiniz
Kontrollü Veri Çekme: Tüm sitelerden veya tek siteden veri çekebilirsiniz
Bekleme Süresi: Siteler arası bekleme süresini ayarlayabilirsiniz
Dosya Kaydetme: Çekilen veriler HTML dosyalarına kaydedilir
Menü Sistemi: Kolay kullanılabilir interaktif menü
Hiçbir ek paket kurmanıza gerek yok!
Uygulama sadece Go'nun standart kütüphanelerini kullanır
go mod init ile projeyi başlatın
Direkt go run main.go ile çalıştırın
