/** * Detik.com Scraper (Search, Latest, Popular & Article Reader) * Author: Vamz Spectre / VamxAPI * * Scraper berita terlengkap detik.com: * - Search Berita (Pencarian kata kunci dengan pagination) * - Berita Terkini (Indeks berita terbaru) * - Berita Terpopuler (Trending / paling banyak dibaca) * - Baca Detail Berita (Judul, Penulis, Tanggal, Gambar Utama, Isi Lengkap Berita) * * Install / Download: * curl -sL "https://code.vamxapi.my.id/snippet/raw/detik-scraper" -o detik-scraper.js * * Usage: * - Search Berita : node detik-scraper.js "prabowo" * - Berita Terkini: node detik-scraper.js --latest * - Berita Populer: node detik-scraper.js --popular * - Detail Berita : node detik-scraper.js "https://news.detik.com/berita/d-8709865/..." */ const cheerio = require('cheerio'); const HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept-Language': 'id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' }; /** * Mencari berita berdasarkan kata kunci * @param {string} query - Kata kunci pencarian * @param {number} [page=1] - Halaman pencarian * @returns {Promise} */ async function search(query, page = 1) { if (!query) throw new Error('Parameter query pencarian tidak boleh kosong.'); const url = `https://www.detik.com/search/searchall?query=${encodeURIComponent(query)}&page=${page}`; const res = await fetch(url, { headers: HEADERS }); if (!res.ok) throw new Error(`Gagal mengambil data pencarian (${res.status} ${res.statusText})`); const html = await res.text(); const $ = cheerio.load(html); const articles = []; $('.list-content__item').each((_, el) => { const linkEl = $(el).find('a[dtr-ttl], .media__link, a').first(); const title = linkEl.attr('dtr-ttl') || $(el).find('h3 a, h2 a, .media__title a').first().text().replace(/\s+/g, ' ').trim() || $(el).find('img').attr('alt') || ''; const articleUrl = $(el).find('a[dtr-ttl], .media__link, a').last().attr('href'); const image = $(el).find('img').attr('src') || null; const category = $(el).find('.media__subtitle, h2').first().text().trim() || 'detikcom'; const date = $(el).find('.media__date span').attr('title') || $(el).find('.media__date').text().trim() || null; const desc = $(el).find('.media__desc').text().replace(/\s+/g, ' ').trim() || null; if (title && articleUrl && articleUrl.startsWith('http')) { articles.push({ title, category, date, description: desc, image, url: articleUrl }); } }); return { status: true, total: articles.length, page: Number(page), query, results: articles }; } /** * Mengambil indeks berita terkini detik.com * @param {number} [page=1] - Halaman indeks * @returns {Promise} */ async function latest(page = 1) { const url = `https://news.detik.com/indeks?page=${page}`; const res = await fetch(url, { headers: HEADERS }); if (!res.ok) throw new Error(`Gagal mengambil berita terkini (${res.status} ${res.statusText})`); const html = await res.text(); const $ = cheerio.load(html); const articles = []; const seenUrls = new Set(); $('.list-content__item, .media').each((_, el) => { const linkEl = $(el).find('a[dtr-ttl], .media__link, a').first(); const title = linkEl.attr('dtr-ttl') || $(el).find('h3 a, h2 a, .media__title a').first().text().replace(/\s+/g, ' ').trim() || $(el).find('img').attr('alt') || ''; const articleUrl = $(el).find('a[dtr-ttl], .media__link, a').last().attr('href'); const image = $(el).find('img').attr('src') || null; const date = $(el).find('.media__date span').attr('title') || $(el).find('.media__date').text().trim() || null; if (title && articleUrl && articleUrl.startsWith('http') && !seenUrls.has(articleUrl)) { seenUrls.add(articleUrl); articles.push({ title, date, image, url: articleUrl }); } }); return { status: true, total: articles.length, page: Number(page), results: articles }; } /** * Mengambil berita terpopuler detik.com * @returns {Promise} */ async function popular() { const url = 'https://www.detik.com/terpopuler'; const res = await fetch(url, { headers: HEADERS }); if (!res.ok) throw new Error(`Gagal mengambil berita populer (${res.status} ${res.statusText})`); const html = await res.text(); const $ = cheerio.load(html); const articles = []; const seenUrls = new Set(); $('.list-content__item, article.list-content__item').each((_, el) => { const linkEl = $(el).find('a[dtr-ttl], .media__link, a').first(); const title = linkEl.attr('dtr-ttl') || $(el).find('h3 a, h2 a, .media__title a').first().text().replace(/\s+/g, ' ').trim() || $(el).find('img').attr('alt') || ''; const articleUrl = $(el).find('a[dtr-ttl], .media__link, a').last().attr('href'); const image = $(el).find('img').attr('src') || null; const category = $(el).find('.media__subtitle, h2').first().text().trim() || 'detikcom'; const date = $(el).find('.media__date span').attr('title') || $(el).find('.media__date').text().trim() || null; if (title && articleUrl && articleUrl.startsWith('http') && !seenUrls.has(articleUrl)) { seenUrls.add(articleUrl); articles.push({ title, category, date, image, url: articleUrl }); } }); return { status: true, total: articles.length, results: articles }; } /** * Membaca detail isi artikel berita lengkap * @param {string} articleUrl - URL artikel berita detik.com * @returns {Promise} */ async function detail(articleUrl) { if (!articleUrl || !articleUrl.startsWith('http')) { throw new Error('Berikan URL artikel berita detik.com yang valid.'); } // Menambahkan ?single=1 agar seluruh halaman berita tampil lengkap dalam 1 request const fetchUrl = articleUrl.includes('?') ? `${articleUrl}&single=1` : `${articleUrl}?single=1`; const res = await fetch(fetchUrl, { headers: HEADERS }); if (!res.ok) throw new Error(`Gagal membuka artikel berita (${res.status} ${res.statusText})`); const html = await res.text(); const $ = cheerio.load(html); const title = $('h1.detail__title, h1.title, .detail__title').first().text().replace(/\s+/g, ' ').trim(); const author = $('.detail__author, .author').first().text().replace(/\s+/g, ' ').trim() || null; const date = $('.detail__date, .date').first().text().replace(/\s+/g, ' ').trim() || null; const image = $('.detail__media-image img, .detail__media img, figure img').first().attr('src') || null; const imageCaption = $('.detail__media-caption, figcaption').first().text().replace(/\s+/g, ' ').trim() || null; // Bersihkan konten dari iklan, tag sisipan, dan script const bodyEl = $('.itp_bodycontent, .detail__body-text, .read__content'); bodyEl.find('script, style, .detail__body-tag, .sisip_video_ds, .paradetiks, .linksisip, div[id^="crosslink"], .banner').remove(); const paragraphs = []; bodyEl.find('p').each((_, p) => { const text = $(p).text().replace(/\s+/g, ' ').trim(); if ( text && !text.toLowerCase().includes('simak juga') && !text.toLowerCase().includes('baca juga') && !text.toUpperCase().includes('SCROLL TO CONTINUE') ) { paragraphs.push(text); } }); const fullContent = paragraphs.length > 0 ? paragraphs.join('\n\n') : bodyEl.text().replace(/\s+/g, ' ').trim(); return { status: true, title, author, date, image, imageCaption, content: fullContent, url: articleUrl }; } module.exports = { search, latest, popular, detail }; // CLI Runner if (require.main === module) { const args = process.argv.slice(2); const input = args[0]; const page = args[1] || 1; if (!input) { console.log( JSON.stringify( { status: false, error: 'Parameter diperlukan.', usage: { search: 'node detik-scraper.js "" [page]', latest: 'node detik-scraper.js --latest [page]', popular: 'node detik-scraper.js --popular', detail: 'node detik-scraper.js ""' } }, null, 2 ) ); process.exit(1); } (async () => { try { let result; if (input === '--latest' || input === '-l') { result = await latest(page); } else if (input === '--popular' || input === '-p') { result = await popular(); } else if (input.startsWith('http://') || input.startsWith('https://')) { result = await detail(input); } else { result = await search(input, page); } console.log(JSON.stringify(result, null, 2)); } catch (err) { console.error( JSON.stringify( { status: false, error: err.message }, null, 2 ) ); process.exit(1); } })(); }