censys tarzı kaynak kod araması yapacak bir script yaptırmak istiyorum.
tahmini 40-50 tb arası bir data içerisinde kelime araması yapacak ve sonuçları listeleyecek bir sistemden bahsediyorum.
elasticsearch kullanılması gerekiyor sanırım.
NOT: data çekecek scraper vs. istemiyorum. dataları kendim çekecegim.
ÖRNEK ÇALIŞMA:
https://fess.codelibs.org/index.html
https://codesearch.codelibs.org/
https://github.com/searchkit/searchkit
https://github.com/appbaseio/dejavu/
https://docs.itemsapi.com/guide
bu çalışmalarda opensource geliştirebilecek olan arkadaşlada çalışabiliriz..
elimdeki datalarda şu şekilde:
{"ip":"138.197.73.171","data":{"banner":{"status":"success","protocol":"banner","result":{"banner":"HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n400 Bad Request","length":103},"timestamp":"2021-08-15T07:23:16Z"},"ftp":{"status":"connection-timeout","protocol":"ftp","timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:21: connect: connection refused"},"http2083":{"status":"connection-timeout","protocol":"http","result":{},"timestamp":"2021-08-15T07:23:18Z","error":"dial tcp \u003cnil\u003e-\u003e138.197.73.171:2083: connect: connection refused"},"http80":{"status":"success","protocol":"http","result":{"response":{"status_line":"404 Not Found","status_code":404,"protocol":{"name":"HTTP/1.1","major":1,"minor":1},"headers":{"content_length":["0"],"date":["Sun, 15 Aug 2021 07:28:58 GMT"],"server":["nginx/1.15.4"]},"request":{"url":{"scheme":"http","host":"138.197.73.171","path":"/"},"method":"GET","headers":{"accept":["*/*"],"user_agent":["Mozilla/5.0 zgrab/0.x"]},"host":"138.197.73.171"}}},"timestamp":"2021-08-15T07:23:18Z"},"http80cpanel":{"status":"success","protocol":"http","result":{"response":{"status_line":"404 Not Found","status_code":404,"protocol":{"name":"HTTP/1.1","major":1,"minor":1},"headers":{"content_length":["0"],"date":["Sun, 15 Aug 2021 07:28:58 GMT"],"server":["nginx/1.15.4"]},"request":{"url":{"scheme":"http","host":"138.197.73.171","path":"/cpanel"},"method":"GET","headers":{"accept":["*/*"],"user_agent":["Mozilla/5.0 zgrab/0.x"]},"host":"138.197.73.171"}}},"timestamp":"2021-08-15T07:23:18Z"},"imap":{"status":"connection-timeout","protocol":"imap","timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:143: connect: connection refused"},"mssql":{"status":"connection-timeout","protocol":"mssql","timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:1433: connect: connection refused"},"mysql":{"status":"connection-timeout","protocol":"mysql","result":null,"timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:3306: connect: connection refused"},"pop3":{"status":"connection-timeout","protocol":"pop3","timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:110: connect: connection refused"},"smtp1":{"status":"connection-timeout","protocol":"smtp","timestamp":"2021-08-15T07:23:18Z","error":"dial tcp 138.197.73.171:25: connect: connection refused"},"smtp2":{"status":"connection-timeout","protocol":"smtp","timestamp":"2021-08-15T07:23:18Z","error":"dial tcp 138.197.73.171:587: connect: connection refused"},"smtp3":{"status":"connection-timeout","protocol":"smtp","timestamp":"2021-08-15T07:23:18Z","error":"dial tcp 138.197.73.171:465: connect: connection refused"},"ssh":{"status":"success","protocol":"ssh","result":{"server_id":{"raw":"SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.2","version":"2.0","software":"OpenSSH_8.2p1","comment":"Ubuntu-4ubuntu0.2"},"server_key_exchange":{"cookie":"gFhHslJXBwCm49mCGWgbpA==","kex_algorithms":["curve25519-sha256","curve25519-sha256@libssh.org","ecdh-sha2-nistp256","ecdh-sha2-nistp384","ecdh-sha2-nistp521","diffie-hellman-group-exchange-sha256","diffie-hellman-group16-sha512","diffie-hellman-group18-sha512","diffie-hellman-group14-sha256"],"host_key_algorithms":["rsa-sha2-512","rsa-sha2-256","ssh-rsa","ecdsa-sha2-nistp256","ssh-ed25519"],"client_to_server_ciphers":["chacha20-poly1305@openssh.com","aes128-ctr","aes192-ctr","aes256-ctr","aes128-gcm@openssh.com","aes256-gcm@openssh.com"],"server_to_client_ciphers":["chacha20-poly1305@openssh.com","aes128-ctr","aes192-ctr","aes256-ctr","aes128-gcm@openssh.com","aes256-gcm@openssh.com"],"client_to_server_macs":["umac-64-etm@openssh.com","umac-128-etm@openssh.com","hmac-sha2-256-etm@openssh.com","hmac-sha2-512-etm@openssh.com","hmac-sha1-etm@openssh.com","umac-64@openssh.com","umac-128@openssh.com","hmac-sha2-256","hmac-sha2-512","hmac-sha1"],"server_to_client_macs":["umac-64-etm@openssh.com","umac-128-etm@openssh.com","hmac-sha2-256-etm@openssh.com","hmac-sha2-512-etm@openssh.com","hmac-sha1-etm@openssh.com","umac-64@openssh.com","umac-128@openssh.com","hmac-sha2-256","hmac-sha2-512","hmac-sha1"],"client_to_server_compression":["none","zlib@openssh.com"],"server_to_client_compression":["none","zlib@openssh.com"],"first_kex_follows":false,"reserved":0},"algorithm_selection":{"dh_kex_algorithm":"curve25519-sha256@libssh.org","host_key_algorithm":"ecdsa-sha2-nistp256","client_to_server_alg_group":{"cipher":"aes128-ctr","mac":"hmac-sha2-256","compression":"none"},"server_to_client_alg_group":{"cipher":"aes128-ctr","mac":"hmac-sha2-256","compression":"none"}},"key_exchange":{"curve25519_sha256_params":{"server_public":"x/Y/6EkmaDPI6KMBkcjszsJyZiW8oWKwK6ihmxlE9Uw="},"server_signature":{"parsed":{"algorithm":"ecdsa-sha2-nistp256","value":"AAAAIH0JCjdg+svw2IxgetB0Y2V0aoQWZ6cZcK7U8b05A4euAAAAIQDp7Rpdr+XmNXF6YcXBVNr4eCgFgRWj++Kfve04XiKFhw=="},"raw":"AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABJAAAAIH0JCjdg+svw2IxgetB0Y2V0aoQWZ6cZcK7U8b05A4euAAAAIQDp7Rpdr+XmNXF6YcXBVNr4eCgFgRWj++Kfve04XiKFhw==","h":"aTT1kjoAzG0lbssuDGRRegHSWlqQZJvA6Zn63bhO7uM="},"server_host_key":{"ecdsa_public_key":{"b":"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","curve":"P-256","gx":"axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","gy":"T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","length":256,"n":"/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","p":"/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","x":"cwOdBuGkl+JjOwCqxffNNgprYWAHE4UcGEsUR0p+aRY=","y":"Fc6GgxPABlSnVvl8/HWf62dJkVF1tj9cvWC8q6A6oDM="},"raw":"AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHMDnQbhpJfiYzsAqsX3zTYKa2FgBxOFHBhLFEdKfmkWFc6GgxPABlSnVvl8/HWf62dJkVF1tj9cvWC8q6A6oDM=","algorithm":"ecdsa-sha2-nistp256","fingerprint_sha256":"4ea6d3e72bb00f8b42d957e91fb364a7dd2ad8d6b3b683f5449da6d255a928a9"}}},"timestamp":"2021-08-15T07:23:16Z"},"telnet":{"status":"connection-timeout","protocol":"telnet","timestamp":"2021-08-15T07:23:17Z","error":"dial tcp 138.197.73.171:23: connect: connection refused"},"tls":{"status":"unknown-error","protocol":"tls","timestamp":"2021-08-15T07:23:16Z","error":"remote error: internal error"}}}iyi forumlar