How to build an Otodom scraper (and other real estate portals)

Each portal has its own structure — we normalize to one:

// schema/listing.ts
import { z } from 'zod';

export const Listing = z.object({
  externalId: z.string(),       // ID from portal
  portal: z.enum(['portal1', 'portal2', 'portal3', 'portal4']),
  url: z.string().url(),
  title: z.string(),
  type: z.enum(['apartment', 'house', 'land', 'commercial']),
  transaction: z.enum(['sale', 'rent']),
  price: z.number().positive(),
  currency: z.enum(['EUR', 'GBP', 'USD']),
  area: z.number().positive(),  // m²
  rooms: z.number().int().nullable(),
  floor: z.number().int().nullable(),
  address: z.object({
    raw: z.string(),
    city: z.string(),
    district: z.string().nullable(),
    street: z.string().nullable(),
    lat: z.number().nullable(),
    lng: z.number().nullable(),
  }),
  description: z.string(),
  images: z.array(z.string().url()),
  scrapedAt: z.date(),
});

Each portal has listing search URLs with pagination:

// portals/portal1.ts
export async function listPortal1Listings(filters) {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  
  const urls = [];
  let pageNum = 1;
  let hasMore = true;
  
  while (hasMore && pageNum <= 100) { // safety limit
    const url = `https://example.com/search?city=${filters.city}&page=${pageNum}`;
    await page.goto(url, { waitUntil: 'networkidle' });
    
    const items = await page.locator('[data-cy="listing-item"]').all();
    for (const item of items) {
      const href = await item.locator('a').first().getAttribute('href');
      urls.push(`https://example.com${href}`);
    }
    
    const nextBtn = page.locator('[data-cy="pagination.next-page"]');
    hasMore = (await nextBtn.count()) > 0 && !(await nextBtn.isDisabled());
    pageNum++;
  }
  
  await browser.close();
  return urls;
}

First we collect URLs (list), then separately scrape details of each. Allows resumption when the process dies.

The same property is often listed on 3-5 portals. Dedup by: address + area + price (with tolerance), or phone number if visible.

async function findDuplicates(listing) {
  // Strategy 1: exact phone match
  if (listing.phone) {
    const byPhone = await db.query(
      'SELECT id, portal FROM listings WHERE phone=$1 AND active=true',
      [listing.phone]
    );
    if (byPhone.rows.length > 0) return byPhone.rows;
  }
  
  // Strategy 2: address + area + price (with 5% tolerance)
  const byAttrs = await db.query(`
    SELECT id, portal, price FROM listings
    WHERE address_normalized = $1
      AND area BETWEEN $2 AND $3
      AND price BETWEEN $4 AND $5
      AND active = true
  `, [
    normalizeAddress(listing.address),
    listing.area * 0.95, listing.area * 1.05,
    listing.price * 0.95, listing.price * 1.05,
  ]);
  
  return byAttrs.rows;
}

async function ingest(listing) {
  const dups = await findDuplicates(listing);
  if (dups.length > 0) {
    // Link as alternative listing
    await db.query(
      'INSERT INTO listing_alternatives (canonical_id, alt_url, portal) VALUES ($1, $2, $3)',
      [dups[0].id, listing.url, listing.portal]
    );
  } else {
    await db.query('INSERT INTO listings (...) VALUES (...)', [...]);
  }
}

Geocode address → lat/lng. Google Maps API ($5/1000 requests) or Nominatim (free, slower):

async function geocode(address) {
  const cached = await db.query(
    'SELECT lat, lng FROM geocode_cache WHERE address=$1', [address]
  );
  if (cached.rows.length) return cached.rows[0];
  
  const url = `https://maps.googleapis.com/maps/api/geocode/json?address=${encodeURIComponent(address)}&key=${API_KEY}`;
  const res = await fetch(url).then(r => r.json());
  if (res.status !== 'OK') return null;
  
  const { lat, lng } = res.results[0].geometry.location;
  await db.query(
    'INSERT INTO geocode_cache (address, lat, lng) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING',
    [address, lat, lng]
  );
  return { lat, lng };
}

Setup PostGIS for geo queries:

CREATE EXTENSION IF NOT EXISTS postgis;
ALTER TABLE listings ADD COLUMN location GEOGRAPHY(POINT, 4326);
UPDATE listings SET location = ST_MakePoint(lng, lat)::geography WHERE lat IS NOT NULL;
CREATE INDEX idx_listings_location ON listings USING GIST (location);

-- Find apartments within 1km of a point
SELECT * FROM listings
WHERE ST_DWithin(location, ST_MakePoint(-0.1, 51.5)::geography, 1000)
  AND price < 800000;

User defines saved search: "apartment, London, 60-80m², up to £850k". After each scrape we match new listings vs searches.

CREATE TABLE saved_searches (
  id BIGSERIAL PRIMARY KEY,
  user_id TEXT NOT NULL,
  filters JSONB NOT NULL,
  notification_channel TEXT, -- 'email' or 'slack'
  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);

-- Match function
CREATE OR REPLACE FUNCTION matches_search(l listings, s saved_searches) RETURNS BOOLEAN AS $$
BEGIN
  IF s.filters->>'type' IS NOT NULL AND l.type != s.filters->>'type' THEN
    RETURN FALSE;
  END IF;
  IF s.filters->>'max_price' IS NOT NULL AND l.price > (s.filters->>'max_price')::numeric THEN
    RETURN FALSE;
  END IF;
  -- ... rest
  RETURN TRUE;
END;
$$ LANGUAGE plpgsql;