How to monitor changes on a website

Simplest approach: fetch text, hash, compare to previous hash. Works great for "did page content change":

import { chromium } from 'playwright';
import crypto from 'node:crypto';

async function getContentHash(url, selector) {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  await page.goto(url, { waitUntil: 'networkidle' });
  const text = await page.locator(selector).innerText();
  await browser.close();
  // Normalize whitespace, strip dynamic timestamps
  const normalized = text.replace(/\s+/g, ' ').trim();
  return {
    hash: crypto.createHash('sha256').update(normalized).digest('hex'),
    text: normalized,
  };
}

async function checkChanges(url, selector) {
  const current = await getContentHash(url, selector);
  const prev = await db.query(
    'SELECT hash, text FROM content_history WHERE url=$1 ORDER BY checked_at DESC LIMIT 1',
    [url]
  );
  if (prev.rows.length === 0 || prev.rows[0].hash !== current.hash) {
    await alertChange(url, prev.rows[0]?.text, current.text);
    await db.query(
      'INSERT INTO content_history (url, hash, text, checked_at) VALUES ($1, $2, $3, NOW())',
      [url, current.hash, current.text]
    );
  }
}

Pitfall: dynamic content (timestamps, ad slots, randomized order) creates false positives. Filter the selector tightly (e.g. main article not body).

Hashing catches content changes but not structural ones (e.g. they added a pricing table, removed a section). DOM diff:

import { diffLines } from 'diff';

async function getDomSnapshot(url, selector) {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  await page.goto(url);
  const html = await page.locator(selector).innerHTML();
  await browser.close();
  // Pretty-print so the diff is readable
  return html.replace(/>\s*</g, '>\n<');
}

async function findStructuralChanges(url, selector) {
  const current = await getDomSnapshot(url, selector);
  const prev = await getPrevSnapshot(url);
  if (!prev) return saveSnapshot(url, current);
  
  const changes = diffLines(prev, current);
  const significant = changes.filter(c => 
    (c.added || c.removed) && c.value.trim().length > 20
  );
  if (significant.length > 0) {
    await alertStructuralChange(url, significant);
  }
}

Hashing and DOM diff miss visual-only changes — e.g. changed banner image, CTA color, layout shift. Screenshot diff:

import pixelmatch from 'pixelmatch';
import { PNG } from 'pngjs';
import fs from 'node:fs';

async function captureAndCompare(url) {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage({ viewport: { width: 1440, height: 900 } });
  await page.goto(url);
  await page.waitForLoadState('networkidle');
  
  const buffer = await page.screenshot({ fullPage: true });
  await browser.close();
  
  const prev = await getPrevScreenshot(url);
  if (!prev) return saveScreenshot(url, buffer);
  
  const img1 = PNG.sync.read(prev);
  const img2 = PNG.sync.read(buffer);
  const { width, height } = img1;
  const diff = new PNG({ width, height });
  
  const numDifferent = pixelmatch(
    img1.data, img2.data, diff.data,
    width, height,
    { threshold: 0.1 } // sensitivity 0-1
  );
  
  const pctChanged = numDifferent / (width * height);
  if (pctChanged > 0.02) { // 2%+ pixels changed
    await alertVisualChange(url, pctChanged, diff);
  }
}

threshold: 0.1 = ignore minor anti-aliasing differences. 2% pixel change threshold = ignore tiny shifts.

Cron or serverless. For 50-200 URLs hourly Vercel cron or Cloudflare Worker suffices.

// Vercel cron: vercel.json
{
  "crons": [
    { "path": "/api/check-changes", "schedule": "0 * * * *" }
  ]
}

// app/api/check-changes/route.ts
export async function GET() {
  const urls = await db.query('SELECT url, selector, technique FROM watched_urls');
  for (const { url, selector, technique } of urls.rows) {
    if (technique === 'hash') await checkChanges(url, selector);
    if (technique === 'dom') await findStructuralChanges(url, selector);
    if (technique === 'screenshot') await captureAndCompare(url);
  }
  return Response.json({ checked: urls.rows.length });
}

Alert template in Slack:

const block = {
  type: 'section',
  text: { type: 'mrkdwn',
    text: `*Change detected*: ${url}\n` +
          `Type: ${technique}\n` +
          `Previous: ${prevSnippet}\n` +
          `Current: ${currentSnippet}`
  }
};