Google Image Botu - "Unusual Traffic" Engeli

Selamlar arkadaşlar,
Python ve Puppeteer stealth eklentisiyle birlikte Google üzerinden ürün görseli çeken bir bot hazırlıyorum. Ancak birkaç sorgudan sonra Google direkt bot olduğumuzu anlıyor ve "unusual traffic" diyerek önümüze duvar örüyor.
Kod tarafında bekleme sürelerini ayarladım, stealth plugin de aktif ama bir yerde patlıyoruz. Bu konuda tecrübesi olan, daha önce bu engeli aşmış ve mantık/yol gösterme konusunda yardımcı olabilecek birileri var mı?
Fikir verebilecek veya "şurayı gözden kaçırıyorsun" diyebilecek arkadaşların yorumlarını bekliyorum. Şimdiden teşekkürler.

Kod:

/**
 * npm i puppeteer-extra puppeteer-extra-plugin-stealth
 * node batch_image_fetcher.js "./products_data.json" "./out"
 */

const fs = require("fs");
const path = require("path");
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());

const delay = (ms) => new Promise((r) => setTimeout(r, ms));
const jitter = (min, max) => min + Math.random() * (max - min);

function escapeRegExp(str) {
  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function normalizeTextTR(s) {
  return (s || "")
    .toLowerCase()
    .replace(/İ/g, "i")
    .replace(/I/g, "i")
    .replace(/ı/g, "i")
    .replace(/\s+/g, " ")
    .trim();
}
function makeExactWordRegex(word) {
  const alphaNumTR = "a-z0-9ğüşıöç";
  const w = escapeRegExp(normalizeTextTR(word));
  return new RegExp(`(^|[^${alphaNumTR}])${w}([^${alphaNumTR}]|$)`, "i");
}
function extractBrand(keyword) {
  const clean = normalizeTextTR(keyword)
    .replace(/[^a-z0-9ğüşıöç\s]/gi, " ")
    .replace(/\s+/g, " ")
    .trim();
  return (clean.split(" ").filter(Boolean)[0] || "").trim();
}
function slugify(s) {
  return normalizeTextTR(s)
    .replace(/[^a-z0-9ğüşıöç\s-]/g, "")
    .replace(/\s+/g, "-")
    .slice(0, 120);
}

async function checkImageUrl(url) {
  if (!url) return false;
  const controller = new AbortController();
  const t = setTimeout(() => controller.abort(), 7000);
  try {
    const res = await fetch(url, {
      method: "GET",
      redirect: "follow",
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        Accept: "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
        Referer: "https://www.google.com/",
      },
      signal: controller.signal,
    });
    if (!res.ok) return false;
    const ct = (res.headers.get("content-type") || "").toLowerCase();
    return ct.includes("image");
  } catch {
    return false;
  } finally {
    clearTimeout(t);
  }
}

async function downloadImage(url, outPath, referer = "https://www.google.com/") {
  const res = await fetch(url, {
    method: "GET",
    redirect: "follow",
    headers: {
      "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
      Accept: "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
      Referer: referer,
    },
  });
  if (!res.ok) throw new Error(`Download failed: ${res.status}`);
  const buf = Buffer.from(await res.arrayBuffer());
  fs.mkdirSync(path.dirname(outPath), { recursive: true });
  fs.writeFileSync(outPath, buf);
}

async function acceptGoogleConsent(page) {
  const selectors = [
    "button#L2AGLb",
    'button[aria-label*="Kabul"]',
    'button[aria-label*="Accept"]',
    'form[action*="consent"] button',
  ];
  for (const sel of selectors) {
    const btn = await page.$(sel);
    if (btn) {
      await btn.click({ delay: 30 });
      await delay(800);
      return true;
    }
  }
  return false;
}
async function isGoogleBlocked(page) {
  const u = page.url();
  const html = (await page.content()).toLowerCase();
  return (
    u.includes("/sorry/") ||
    html.includes("unusual traffic") ||
    html.includes("our systems have detected")
  );
}

async function waitForGoogleGrid(page) {
  await page.waitForSelector("body", { timeout: 30000 });
  await acceptGoogleConsent(page);
  await page.waitForFunction(() => {
    return (
      document.querySelectorAll("img.Q4LuWd").length > 0 ||
      document.querySelectorAll("img.rg_i").length > 0 ||
      document.querySelectorAll('div[data-ri] img').length > 0
    );
  }, { timeout: 45000 });
}

async function googleFetchOne(page, keyword, outPath) {
  const brand = extractBrand(keyword);
  const brandRx = makeExactWordRegex(brand);

  const url = `https://www.google.com/search?tbm=isch&q=${encodeURIComponent(keyword)}`;
  await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });

  if (await isGoogleBlocked(page)) {
    return { ok: false, reason: "google_blocked" };
  }

  await waitForGoogleGrid(page);
  if (await isGoogleBlocked(page)) {
    return { ok: false, reason: "google_blocked" };
  }

  await page.evaluate(() => window.scrollBy(0, 700));
  await delay(jitter(500, 900));

  const gridImgs = await page.$$("img.Q4LuWd, img.rg_i, div[data-ri] img");
  const tries = Math.min(25, gridImgs.length);

  for (let i = 0; i < tries; i++) {
    const img = gridImgs[i];
    try {
      const box = await img.boundingBox();
      if (!box || box.width < 80 || box.height < 80) continue;
    } catch {}

    try {
      await img.click({ delay: 20 });
    } catch {
      continue;
    }

    await delay(jitter(700, 1100));

    // caption signals
    let captionOk = false;
    for (let k = 0; k < 5; k++) {
      const signals = await page.evaluate(() => {
        const out = new Set();
        const dialog =
          document.querySelector('[role="dialog"]') ||
          document.querySelector("c-wiz[role]") ||
          document.querySelector("c-wiz") ||
          document.body;

        const els = Array.from(dialog.querySelectorAll("h1,h2,h3,a,span,div")).slice(0, 700);
        for (const el of els) {
          const t = (el.innerText || "").trim();
          if (!t) continue;
          if (t.length < 3 || t.length > 90) continue;
          out.add(t);
        }
        return Array.from(out);
      });

      const norm = signals.map(normalizeTextTR);
      captionOk = norm.some((t) => brandRx.test(t));
      if (captionOk) break;
      await delay(jitter(200, 450));
    }

    if (!captionOk) {
      try { await page.keyboard.press("Escape"); } catch {}
      await delay(jitter(120, 250));
      continue;
    }

    // collect preview candidates
    const previewUrls = await page.evaluate(() => {
      const urls = new Set();
      const primary = Array.from(document.querySelectorAll("img.n3VNCb"));
      const imgs = primary.length ? primary : Array.from(document.querySelectorAll("img"));

      for (const img of imgs) {
        const w = img.naturalWidth || img.width || 0;
        const h = img.naturalHeight || img.height || 0;
        if (w && h && (w < 220 || h < 220)) continue;

        const src = img.currentSrc || img.src || "";
        if (src && src.startsWith("http")) urls.add(src);

        const srcset = img.getAttribute("srcset") || "";
        if (srcset) {
          const parts = srcset
            .split(",")
            .map((p) => p.trim().split(/\s+/)[0])
            .filter((u) => u && u.startsWith("http"));
          for (const u of parts) urls.add(u);
        }
      }
      return Array.from(urls).slice(0, 80);
    });

    // download first working
    for (const u of previewUrls) {
      const alive = await checkImageUrl(u);
      if (!alive) continue;
      await downloadImage(u, outPath, "https://www.google.com/");
      return { ok: true, engine: "google", imageUrl: u, brand };
    }

    try { await page.keyboard.press("Escape"); } catch {}
    await delay(jitter(120, 250));
  }

  return { ok: false, reason: "google_no_match" };
}

/**
 * Bing fallback (free, usually less aggressive blocks)
 * We enforce brand exact word on result title/alt text.
 */
async function bingFetchOne(page, keyword, outPath) {
  const brand = extractBrand(keyword);
  const brandRx = makeExactWordRegex(brand);

  const url = `https://www.bing.com/images/search?q=${encodeURIComponent(keyword)}`;
  await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });

  await page.waitForSelector("body", { timeout: 30000 });
  await delay(jitter(500, 900));

  // Bing result thumbs typically: a.iusc (data-m) with metadata
  await page.waitForFunction(() => {
    return document.querySelectorAll("a.iusc").length > 0 || document.querySelectorAll("img.mimg").length > 0;
  }, { timeout: 45000 });

  // collect candidates with metadata
  const candidates = await page.evaluate(() => {
    const out = [];
    const items = Array.from(document.querySelectorAll("a.iusc")).slice(0, 40);
    for (const a of items) {
      const m = a.getAttribute("m"); // json-ish metadata
      const img = a.querySelector("img");
      const alt = (img?.getAttribute("alt") || "").trim();
      out.push({ m, alt });
    }
    return out;
  });

  for (const c of candidates) {
    const text = normalizeTextTR(c.alt || "");
    if (!brandRx.test(text)) continue;

    let mObj = null;
    try { mObj = c.m ? JSON.parse(c.m) : null; } catch {}
    const murl = mObj?.murl; // full image url
    if (!murl) continue;

    const alive = await checkImageUrl(murl);
    if (!alive) continue;

    await downloadImage(murl, outPath, "https://www.bing.com/");
    return { ok: true, engine: "bing", imageUrl: murl, brand };
  }

  return { ok: false, reason: "bing_no_match" };
}

async function main(jsonPath, outDir) {
  const raw = fs.readFileSync(jsonPath, "utf-8");
  const data = JSON.parse(raw);
  const products = data.products || [];

  fs.mkdirSync(outDir, { recursive: true });

  const browser = await puppeteer.launch({
    headless: false, // insan-onaylı akış; blok olursa görürsün
    args: [
      "--no-sandbox",
      "--disable-setuid-sandbox",
      "--disable-dev-shm-usage",
      "--window-size=1920,1080",
      "--user-data-dir=./chrome-profile",
      "--profile-directory=Default",
      "--lang=tr-TR,tr",
    ],
  });

  try {
    const page = await browser.newPage();
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent(
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    );
    await page.setExtraHTTPHeaders({
      "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
    });

    // block css/fonts for speed
    await page.setRequestInterception(true);
    page.on("request", (req) => {
      const t = req.resourceType();
      if (t === "stylesheet" || t === "font") return req.abort();
      return req.continue();
    });

    const results = [];

    for (let idx = 0; idx < products.length; idx++) {
      const p = products[idx];
      const name = p.name || `product-${idx + 1}`;
      const brand = extractBrand(name);
      const fileBase =
        (p.barcode && p.barcode.trim()) ||
        (p.sku && p.sku.trim()) ||
        `${idx + 1}-${slugify(name)}`;

      const outPath = path.join(outDir, `${fileBase}.jpg`);

      // polite delay between products
      await delay(jitter(3000, 8000));

      // 1) Google try with retries/backoff (no bypass)
      let r = null;
      for (let attempt = 1; attempt <= 3; attempt++) {
        r = await googleFetchOne(page, name, outPath);

        if (r.ok) break;

        if (r.reason === "google_blocked") {
          // backoff (give it time)
          const waitMs = jitter(60000, 180000);
          console.log(JSON.stringify({ name, attempt, blocked: true, waitMs: Math.round(waitMs) }));
          await delay(waitMs);
          continue;
        }

        // no_match -> small wait and retry once
        await delay(jitter(5000, 12000));
      }

      // 2) Fallback to Bing
      if (!r || !r.ok) {
        const b = await bingFetchOne(page, name, outPath);
        if (b.ok) r = b;
      }

      results.push({
        name,
        brand,
        outPath,
        status: r?.ok ? "ok" : "fail",
        engine: r?.engine || null,
        imageUrl: r?.imageUrl || null,
        reason: r?.ok ? null : r?.reason || "no_result",
      });

      console.log(JSON.stringify(results[results.length - 1]));
    }

    fs.writeFileSync(
      path.join(outDir, "results.json"),
      JSON.stringify({ results }, null, 2),
      "utf-8"
    );

    console.log(JSON.stringify({ done: true, count: results.length, resultsFile: path.join(outDir, "results.json") }));
  } finally {
    await browser.close();
  }
}

const jsonPath = process.argv[2];
const outDir = process.argv[3] || "./out";

if (!jsonPath) {
  console.log("Usage: node batch_image_fetcher.js ./products_data.json ./out");
  process.exit(1);
}

main(jsonPath, outDir).catch((e) => {
  console.log(JSON.stringify({ error: e.message }));
  process.exit(1);
});

telegram: https://t.me/r10wisex

boşverin sıfırdan bot yazacağım diye uğraşmayı, bu Google nasıl atlatılacak diye kafa yormayı, dilediğiniz gibi özelleştirebileceğiniz hazır altyapısı mevcut, ÜCRETSİZ kullanım seçeneği ile ki sizin işinizi ücretsiz versiyon da çözüyor, tek sayfa modül yazacaksınız ki elinizde zaten Google Image'den resim çeken kod mevcut, bu kısmı UWA'ya mission modülü olarak yazıp koyacaksınız UWA-User dizinine ve iş tamamdır: https://www.r10.net/bot-program-sati...a-fazlasi.html