diff --git a/README.md b/README.md index e6ea0e0..25c4a03 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ | `brightdata scraper heal` | Fix an existing scraper in place via AI self-healing (stops at an approval gate) | | `brightdata scraper approve` | Approve (or reject) a self-healing fix that is awaiting approval | | `brightdata pipelines` | Extract structured data from 40+ platforms (Amazon, LinkedIn, TikTok…) | +| `brightdata marketplace` | Query Bright Data's pre-collected datasets — filter records that already exist | | `brightdata browser` | Control a real browser via Bright Data's Scraping Browser — navigate, snapshot, click, type, and more | | `brightdata zones` | List and inspect your Bright Data proxy zones | | `brightdata budget` | View account balance and per-zone cost & bandwidth | @@ -56,6 +57,7 @@ - [scraper heal](#scraper-heal) - [scraper approve](#scraper-approve) - [pipelines](#pipelines) + - [marketplace](#marketplace) - [browser](#browser) - [status](#status) - [zones](#zones) @@ -653,6 +655,82 @@ See [Dataset Types Reference](#dataset-types-reference) for the full list. --- +### `marketplace` + +Query Bright Data's **Dataset Marketplace** — data that has already been collected. This is the counterpart to [`pipelines`](#pipelines), and the distinction matters for both speed and cost: + +| | `pipelines` | `marketplace` | +|---|---|---| +| What it does | **Collects new** data from URLs/seeds you supply | **Queries** records Bright Data already holds | +| Billing | Per record scraped | Per record returned by the query | +| Latency | A crawl runs | A filter runs over stored data | + +```bash +brightdata marketplace list [--featured] [--search ] +brightdata marketplace fields +brightdata marketplace filter --dataset --filter '' --records-limit +brightdata marketplace status +brightdata marketplace download [--wait] +``` + +**Finding a dataset.** The catalogue holds ~1,750 datasets, so `list` has two filters. `--featured` shows the 48 with short names you can type; `--search` finds anything else, and you pass its `gd_` id to the other subcommands with `--dataset-id`. + +```bash +brightdata marketplace list --featured +brightdata marketplace list --search zillow +``` + +**Seeing what you can filter on** — before spending anything: + +```bash +brightdata marketplace fields linkedin_people_profiles +# field type required description +# about text A concise profile summary... +# city text Geographical location of the user +``` + +**Querying.** The filter is a JSON tree, sent to the API as-is: + +```bash +brightdata marketplace filter --dataset linkedin_company_profiles \ + --filter '{"name":"industry","operator":"=","value":"Technology"}' \ + --records-limit 1000 --format jsonl -o tech.jsonl + +# combine conditions with and/or +--filter '{"operator":"and","filters":[ + {"name":"industry","operator":"=","value":"Technology"}, + {"name":"followers","operator":">","value":10000}]}' +``` + +> **`--records-limit` is required.** Queries are billed by records returned, some datasets hold hundreds of millions of rows, and the cost is only known once the query has run — so the CLI makes you state a cap rather than letting an omission become an expensive one. + +**Submit now, collect later:** + +```bash +brightdata marketplace filter --dataset x_twitter_posts \ + --filter '{"name":"likes","operator":">","value":10000}' \ + --records-limit 500 --async +# → Snapshot ID: s_abc123 + +brightdata marketplace status s_abc123 # status, records, size, cost +brightdata marketplace download s_abc123 --wait +``` + +`download` exit codes: `0` data returned · `1` error · `3` snapshot not ready yet (retry later, or use `--wait`). + +| Flag | Applies to | Description | +|---|---|---| +| `--featured` / `--search ` | `list` | Filter the catalogue | +| `--dataset ` | `filter` | One of the 48 named datasets | +| `--dataset-id ` | `filter` | Any dataset id from `list` | +| `--filter ` / `--filter-file ` | `filter` | The filter tree | +| `--records-limit ` | `filter` | **Required** — cap on records returned | +| `--async` | `filter` | Submit only; print the snapshot ID | +| `--wait` | `download` | Poll until ready, then download | +| `--format ` | `filter`, `download` | `json` · `jsonl` · `csv` (no `ndjson`) | +| `-o, --output ` / `--json` / `--pretty` | all | Output handling | + + ### `browser` Control a real browser session powered by [Bright Data's Scraping Browser](https://brightdata.com/products/scraping-browser). A lightweight local daemon holds the browser connection open between commands, giving you persistent state without reconnecting on every call. diff --git a/src/__tests__/commands/marketplace.test.ts b/src/__tests__/commands/marketplace.test.ts new file mode 100644 index 0000000..63d2a0f --- /dev/null +++ b/src/__tests__/commands/marketplace.test.ts @@ -0,0 +1,367 @@ +import {describe, it, expect, beforeEach, afterEach, vi} from 'vitest'; + +const mocks = vi.hoisted(()=>({ + get: vi.fn(), + post: vi.fn(), + ensure_authenticated: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + print: vi.fn(), + print_table: vi.fn(), + info: vi.fn(), + success: vi.fn(), + warn: vi.fn(), + fail: vi.fn((msg: string)=>{ throw new Error(`fail:${msg}`); }), + parse_timeout: vi.fn(), + poll_until: vi.fn(), +})); + +vi.mock('../../utils/client', ()=>({get: mocks.get, post: mocks.post})); +vi.mock('../../utils/auth', ()=>({ + ensure_authenticated: mocks.ensure_authenticated, +})); +vi.mock('../../utils/spinner', ()=>({start: mocks.start})); +vi.mock('../../utils/output', ()=>({ + print: mocks.print, + print_table: mocks.print_table, + dim: (s: string)=>s, + fail: mocks.fail, + info: mocks.info, + success: mocks.success, + warn: mocks.warn, +})); +vi.mock('../../utils/polling', ()=>({ + parse_timeout: mocks.parse_timeout, + poll_until: mocks.poll_until, +})); + +import { + handle_list, + handle_fields, + handle_filter, + handle_status, + handle_download, + FEATURED_DATASETS, +} from '../../commands/marketplace'; + +const LINKEDIN_PEOPLE = 'gd_l1viktl72bvl7bjuj0'; + +describe('commands/marketplace', ()=>{ + let exit_spy: ReturnType; + + beforeEach(()=>{ + vi.clearAllMocks(); + mocks.ensure_authenticated.mockReturnValue('api_key'); + mocks.start.mockReturnValue({stop: mocks.stop}); + mocks.parse_timeout.mockReturnValue(600); + exit_spy = vi.spyOn(process, 'exit') + .mockImplementation(((..._a: unknown[])=>undefined) as never); + vi.spyOn(console, 'error').mockImplementation(()=>{}); + }); + + afterEach(()=>{ + vi.restoreAllMocks(); + }); + + describe('the curated alias map', ()=>{ + it('covers 48 datasets and every id looks like a dataset id', ()=>{ + const entries = Object.entries(FEATURED_DATASETS); + expect(entries).toHaveLength(48); + for (const [, id] of entries) + expect(id).toMatch(/^gd_[a-z0-9]+$/); + }); + + it('maps no two names to the same id', ()=>{ + const ids = Object.values(FEATURED_DATASETS); + expect(new Set(ids).size).toBe(ids.length); + }); + }); + + describe('list', ()=>{ + it('fetches the catalogue and renders a table', async()=>{ + mocks.get.mockResolvedValue([ + {id: 'gd_a', name: 'Alpha ', size: 1000}, + ]); + await handle_list({}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/list', {timing: undefined}); + // the catalogue carries stray whitespace; display trims it + expect(mocks.print_table).toHaveBeenCalledWith( + [{name: 'Alpha', id: 'gd_a', records: '1,000'}], + ['name', 'id', 'records'] + ); + }); + + it('--featured keeps only curated datasets', async()=>{ + mocks.get.mockResolvedValue([ + {id: LINKEDIN_PEOPLE, name: 'LinkedIn people profiles'}, + {id: 'gd_not_curated', name: 'Something else'}, + ]); + await handle_list({featured: true}); + const rows = mocks.print_table.mock.calls[0][0]; + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(LINKEDIN_PEOPLE); + }); + + it('--search matches name or id, case-insensitively', async()=>{ + mocks.get.mockResolvedValue([ + {id: 'gd_a', name: 'Zillow properties'}, + {id: 'gd_zillow_b', name: 'Something'}, + {id: 'gd_c', name: 'Unrelated'}, + ]); + await handle_list({search: 'ZILLOW'}); + expect(mocks.print_table.mock.calls[0][0]).toHaveLength(2); + }); + + it('says so when nothing matches instead of printing nothing', + async()=>{ + // print_table returns silently on an empty array, so an empty + // result would otherwise be indistinguishable from a failure. + mocks.get.mockResolvedValue([{id: 'gd_a', name: 'Alpha'}]); + await handle_list({search: 'nothing-matches-this'}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('No datasets match')); + expect(mocks.print_table).not.toHaveBeenCalled(); + }); + + it('passes --json straight through to print', async()=>{ + const rows = [{id: 'gd_a', name: 'Alpha'}]; + mocks.get.mockResolvedValue(rows); + await handle_list({json: true}); + expect(mocks.print).toHaveBeenCalledWith( + rows, {json: true, pretty: undefined, output: undefined}); + expect(mocks.print_table).not.toHaveBeenCalled(); + }); + }); + + describe('fields', ()=>{ + it('resolves a curated alias without touching the catalogue', + async()=>{ + mocks.get.mockResolvedValue({fields: {url: {type: 'url'}}}); + await handle_fields('linkedin_people_profiles', {}); + expect(mocks.get).toHaveBeenCalledTimes(1); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', + `/datasets/${LINKEDIN_PEOPLE}/metadata`, + {timing: undefined} + ); + }); + + it('accepts a raw gd_ id', async()=>{ + mocks.get.mockResolvedValue({fields: {a: {type: 'text'}}}); + await handle_fields('gd_anything', {}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/gd_anything/metadata', + {timing: undefined}); + }); + + it('rejects an unknown name and points at list/search', async()=>{ + await expect(handle_fields('not_a_dataset', {})) + .rejects.toThrow(/Unknown dataset "not_a_dataset"/); + expect(mocks.fail).toHaveBeenCalledWith( + expect.stringContaining('--search not_a_dataset')); + expect(mocks.get).not.toHaveBeenCalled(); + }); + + it('truncates long descriptions for the table but not for --json', + async()=>{ + const long = 'x'.repeat(200); + mocks.get.mockResolvedValue({fields: {a: {description: long}}}); + await handle_fields('gd_x', {}); + const row = mocks.print_table.mock.calls[0][0][0]; + expect(row.description.length).toBeLessThan(80); + expect(row.description.endsWith('…')).toBe(true); + }); + }); + + describe('filter — the billable path', ()=>{ + const base = { + dataset: 'crunchbase_companies', + filter: '{"name":"industry","operator":"=","value":"Tech"}', + recordsLimit: '100', + }; + + it('refuses to run without --records-limit', async()=>{ + // The billing guard: omitting the cap means "no limit" on datasets + // holding hundreds of millions of records, and cost is only known + // after the query is committed. + await expect(handle_filter({ + dataset: base.dataset, + filter: base.filter, + })).rejects.toThrow(/--records-limit is required/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('rejects a non-positive --records-limit', async()=>{ + await expect(handle_filter({...base, recordsLimit: '0'})) + .rejects.toThrow(/Invalid --records-limit/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('sends dataset_id, the parsed filter, and the cap', async()=>{ + mocks.post.mockResolvedValue({snapshot_id: 's_1'}); + await handle_filter({...base, async: true}); + expect(mocks.post).toHaveBeenCalledWith( + 'api_key', + '/datasets/filter', + { + dataset_id: FEATURED_DATASETS['crunchbase_companies'], + filter: { + name: 'industry', operator: '=', value: 'Tech', + }, + records_limit: 100, + }, + {timing: undefined} + ); + }); + + it('fails before the network on invalid filter JSON', async()=>{ + await expect(handle_filter({...base, filter: 'not json'})) + .rejects.toThrow(/Invalid JSON in filter/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('rejects --filter and --filter-file together', async()=>{ + await expect(handle_filter({ + ...base, filterFile: 'f.json', + })).rejects.toThrow(/not both/); + expect(mocks.post).not.toHaveBeenCalled(); + }); + + it('surfaces the API error when no snapshot_id comes back', + async()=>{ + mocks.post.mockResolvedValue({error: 'bad filter field'}); + await expect(handle_filter(base)) + .rejects.toThrow(/bad filter field/); + }); + + it('--async prints the id and does not poll', async()=>{ + mocks.post.mockResolvedValue({snapshot_id: 's_1'}); + await handle_filter({...base, async: true}); + expect(mocks.success).toHaveBeenCalledWith( + expect.stringContaining('s_1')); + expect(mocks.poll_until).not.toHaveBeenCalled(); + }); + }); + + describe('status', ()=>{ + it('surfaces cost, record count and file size', async()=>{ + mocks.get.mockResolvedValue({ + status: 'ready', dataset_size: 1234, file_size: 2097152, + cost: 4.5, + }); + await handle_status('s_1', {}); + expect(mocks.get).toHaveBeenCalledWith( + 'api_key', '/datasets/snapshots/s_1', {timing: undefined}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('cost 4.5')); + }); + + it('fails loudly on a failed snapshot', async()=>{ + mocks.get.mockResolvedValue({ + status: 'failed', failure_reason: 'filter referenced no field', + }); + await expect(handle_status('s_1', {})) + .rejects.toThrow(/filter referenced no field/); + }); + }); + + describe('download', ()=>{ + it('exits NOT_READY (3) when still building and no --wait', + async()=>{ + mocks.get.mockResolvedValue({status: 'building'}); + await handle_download('s_1', {}); + expect(exit_spy).toHaveBeenCalledWith(3); + expect(mocks.poll_until).not.toHaveBeenCalled(); + expect(mocks.print).not.toHaveBeenCalled(); + }); + + it('downloads a ready snapshot as parsed json', async()=>{ + mocks.get + .mockResolvedValueOnce({status: 'ready', dataset_size: 2}) + .mockResolvedValueOnce([{a: 1}]); + await handle_download('s_1', {format: 'json'}); + expect(mocks.get).toHaveBeenLastCalledWith( + 'api_key', + '/datasets/snapshots/s_1/download?format=json', + {timing: undefined} + ); + expect(mocks.print).toHaveBeenCalledWith( + [{a: 1}], + {json: undefined, pretty: undefined, output: undefined} + ); + }); + + it('requests non-json formats as raw bytes, not parsed json', + async()=>{ + // The endpoint returns content-type application/jsonl, which + // the client's `includes('application/json')` check would + // treat as JSON — parsing the body and re-serialising it as + // indented JSON. raw_buffer keeps it byte-exact. + mocks.get + .mockResolvedValueOnce({status: 'ready'}) + .mockResolvedValueOnce(Buffer.from('{"a":1}\n')); + await handle_download('s_1', {format: 'jsonl'}); + expect(mocks.get).toHaveBeenLastCalledWith( + 'api_key', + '/datasets/snapshots/s_1/download?format=jsonl', + {timing: undefined, raw_buffer: true} + ); + expect(mocks.print).toHaveBeenCalledWith( + '{"a":1}\n', + {json: undefined, pretty: undefined, output: undefined} + ); + }); + + it('reports a zero-match query as an empty result, not a failure', + async()=>{ + // A filter matching nothing comes back as status:failed with + // warning_code no_records_found — a successful empty query. + mocks.get.mockResolvedValue({ + status: 'failed', + warning: 'Provided filter did not match any records', + warning_code: 'no_records_found', + }); + await handle_download('s_1', {}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('matched no records')); + expect(exit_spy).not.toHaveBeenCalled(); + }); + + it('--wait polls on the marketplace status vocabulary', async()=>{ + mocks.get.mockResolvedValueOnce({status: 'scheduled'}); + mocks.poll_until.mockResolvedValue({ + result: {status: 'ready'}, attempts: 2, + }); + mocks.get.mockResolvedValueOnce([{a: 1}]); + await handle_download('s_1', {wait: true}); + expect(mocks.poll_until).toHaveBeenCalledWith( + expect.objectContaining({ + running_statuses: ['scheduled', 'building'], + }) + ); + }); + + it('reports an empty snapshot instead of printing nothing', + async()=>{ + mocks.get + .mockResolvedValueOnce({status: 'ready'}) + .mockResolvedValueOnce([]); + await handle_download('s_1', {}); + expect(mocks.info).toHaveBeenCalledWith( + expect.stringContaining('No records')); + expect(mocks.print).not.toHaveBeenCalled(); + }); + + it('fails with the API reason on a failed snapshot', async()=>{ + mocks.get.mockResolvedValue({status: 'failed', error: 'boom'}); + await expect(handle_download('s_1', {})).rejects.toThrow(/boom/); + }); + + it('rejects ndjson, which is a pipelines format', async()=>{ + await expect(handle_download('s_1', {format: 'ndjson'})) + .rejects.toThrow(/Invalid format "ndjson"/); + expect(mocks.get).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/commands/marketplace.ts b/src/commands/marketplace.ts new file mode 100644 index 0000000..705f9d2 --- /dev/null +++ b/src/commands/marketplace.ts @@ -0,0 +1,749 @@ +import fs from 'fs'; +import {Command} from 'commander'; +import {ensure_authenticated} from '../utils/auth'; +import {get, post} from '../utils/client'; +import { + print, print_table, dim, fail, info, success, +} from '../utils/output'; +import {start as start_spinner} from '../utils/spinner'; +import {parse_timeout, poll_until} from '../utils/polling'; +import {add_examples} from '../utils/help'; +import {EXIT} from '../utils/exit-codes'; +import type { + Marketplace_format, + Dataset_info, + Dataset_metadata, + Snapshot_status, + Filter_request, + Filter_response, + Marketplace_opts, +} from '../types/marketplace'; + +// The Dataset Marketplace API: query records Bright Data has already +// collected. Unversioned, and entirely separate from the /datasets/v3/* +// endpoints `pipelines` uses to collect new data. +const LIST_ENDPOINT = '/datasets/list'; +const FILTER_ENDPOINT = '/datasets/filter'; +const metadata_endpoint = (id: string)=>`/datasets/${id}/metadata`; +const snapshot_endpoint = (id: string)=>`/datasets/snapshots/${id}`; +const download_endpoint = (id: string)=>`/datasets/snapshots/${id}/download`; + +// Marketplace status vocabulary — differs from v3's +// starting/building/running. +const RUNNING_STATUSES = ['scheduled', 'building']; +const FAILED_STATUS = 'failed'; +const ALLOWED_FORMATS: Marketplace_format[] = ['json', 'jsonl', 'csv']; +const LIST_WARN_THRESHOLD = 200; + +// Named shortcuts for the datasets people reach for most. This is the ONLY way +// to address a dataset by name: the catalogue's own `name` field is a human +// display string ("Instagram - Profiles", "Facebook - Comments" with a double +// space, "Manta businesses " with a trailing one), 43 names are shared across +// different ids, and the catalogue holds internal entries. So names are for +// reading, not for resolving. Everything outside this map is reachable with +// --dataset-id; `marketplace list --search` finds the id. +// Each comment records the catalogue's own name for that id (validated against +// a live GET /datasets/list on 2026-08-27: 48/48 resolve). +const FEATURED_DATASETS: Record = { + // --- social / content (32) --- + // Bluesky - Posts + bluesky_posts: 'gd_m6hn4r5s27zfhc7w4', + // Top 500 Bluesky Profiles + bluesky_top_profiles: 'gd_m45p78dl1m017wi5lj', + // Facebook - Comments + facebook_comments: 'gd_lkay758p1eanlolqw8', + // Facebook Company Reviews + facebook_company_reviews: 'gd_m0dtqpiu1mbcyc2g86', + // Facebook Events + facebook_events: 'gd_m14sd0to1jz48ppm51', + // Facebook - Posts by group URL + facebook_group_posts: 'gd_lz11l67o2cb3r0lkj3', + // Facebook Marketplace + facebook_marketplace: 'gd_lvt9iwuh6fbcwmx1a', + // Facebook - Pages Posts by Profile URL + facebook_pages_posts: 'gd_lkaxegm826bjpoo9m5', + // Facebook - Pages and Profiles + facebook_pages_profiles: 'gd_mf124a0511bauquyow', + // Facebook - Posts by post URL + facebook_posts_by_url: 'gd_lyclm1571iy3mv57zw', + // Facebook - Profiles + facebook_profiles: 'gd_mf0urb782734ik94dz', + // Facebook - Reels by profile URL + facebook_reels: 'gd_lyclm3ey2q6rww027t', + // Instagram - Comments + instagram_comments: 'gd_ltppn085pokosxh13', + // Instagram - Posts + instagram_posts: 'gd_lk5ns7kz21pck8jpis', + // Instagram - Profiles + instagram_profiles: 'gd_l1vikfch901nx3by4', + // Instagram - Reels + instagram_reels: 'gd_lyclm20il4r5helnj', + // Pinterest - Posts + pinterest_posts: 'gd_lk0sjs4d21kdr7cnlv', + // Pinterest - Profiles + pinterest_profiles: 'gd_lk0zv93c2m9qdph46z', + // Quora posts + quora_posts: 'gd_lvz1rbj81afv3m6n5y', + // Reddit - Comments + reddit_comments: 'gd_lvzdpsdlw09j6t702', + // Reddit- Posts + reddit_posts: 'gd_lvz8ah06191smkebj4', + // Snapchat posts + snapchat_posts: 'gd_ma0ydx431w6stl16ge', + // TikTok - Comments + tiktok_comments: 'gd_lkf2st302ap89utw5k', + // TikTok - Posts + tiktok_posts: 'gd_lu702nij2f790tmv9h', + // TikTok - Profiles + tiktok_profiles: 'gd_l1villgoiiidt09ci', + // TikTok Shop + tiktok_shop: 'gd_m45m1u911dsa4274pi', + // Vimeo - Videos posts + vimeo_videos: 'gd_lxk88z3v1ketji4pn', + // X (formerly Twitter) - Posts + x_twitter_posts: 'gd_lwxkxvnf1cynvib9co', + // X (formerly Twitter) - Profiles + x_twitter_profiles: 'gd_lwxmeb2u1cniijd7t4', + // Youtube - Comments + youtube_comments: 'gd_lk9q0ew71spt1mxywf', + // YouTube - Channels + youtube_profiles: 'gd_lk538t2k2p1k3oos71', + // Youtube - Videos posts + youtube_videos: 'gd_lk56epmy2i5g7lzu0k', + // --- business / people intelligence (16) --- + // Companies information enriched dataset + companies_enriched: 'gd_m3fl0mwzmfpfn4cw4', + // Crunchbase companies information + crunchbase_companies: 'gd_l1vijqt9jfj7olije', + // Employees business enriched dataset + employees_enriched: 'gd_m18zt6ec11wfqohyrs', + // LinkedIn company information + linkedin_company_profiles: 'gd_l1vikfnt1wgvvqz95w', + // Linkedin job listings information + linkedin_job_listings: 'gd_lpfll7v5hcqtkxl6l', + // LinkedIn people profiles + linkedin_people_profiles: 'gd_l1viktl72bvl7bjuj0', + // LinkedIn posts + linkedin_posts: 'gd_lyy3tktm25m4avu764', + // LinkedIn profiles Jobs Listings + linkedin_profiles_job_listings: 'gd_m487ihp32jtc4ujg45', + // Manta businesses + manta_businesses: 'gd_l1vil1d81g0u8763b2', + // Owler companies information + owler_companies: 'gd_l1vilaxi10wutoage7', + // pitchbook companies information + pitchbook_companies: 'gd_m4ijiqfp2n9oe3oluj', + // Slintel 6sense company information + slintel_companies: 'gd_l1vilg5a1decoahvgq', + // US lawyers directory + us_lawyers: 'gd_l1vil5n11okchcbvax', + // VentureRadar company information + ventureradar_companies: 'gd_l1vilsfd1xpsndbtpr', + // Xing social network + xing_profiles: 'gd_l3lh4ev31oqrvvblv6', + // Zoominfo companies information + zoominfo_companies: 'gd_m0ci4a4ivx3j5l6nx', +}; + +const resolve_format = (raw: string|undefined): Marketplace_format=>{ + const format = (raw ?? 'json').toLowerCase(); + if (ALLOWED_FORMATS.includes(format as Marketplace_format)) + return format as Marketplace_format; + fail( + `Invalid format "${format}".\n` + +' Allowed formats: json, jsonl, csv.\n' + +' (ndjson is a pipelines format; the marketplace API does not ' + +'accept it.)' + ); + return 'json'; +}; + +// gd_ id -> used as-is; curated alias -> its id; anything else -> fail with a +// pointer. There is deliberately no lookup against the catalogue's `name` +// field: it is a display string, not an identifier, and 43 names are shared +// across ids, so a "match" could not identify one dataset anyway. +const resolve_dataset_id = (opts: Marketplace_opts): string|undefined=>{ + if (opts.dataset && opts.datasetId) + { + fail('Provide either --dataset or --dataset-id, not both.'); + return undefined; + } + if (opts.datasetId) + return opts.datasetId.trim(); + const ref = opts.dataset?.trim(); + if (!ref) + { + fail( + 'No dataset specified.\n' + +' Use --dataset (see \'brightdata marketplace list ' + +'--featured\')\n' + +' or --dataset-id for any other dataset.' + ); + return undefined; + } + if (ref.startsWith('gd_')) + return ref; + const featured = FEATURED_DATASETS[ref.toLowerCase()]; + if (featured) + return featured; + fail( + `Unknown dataset "${ref}".\n` + +' Named datasets: brightdata marketplace list --featured\n' + +` Anything else: brightdata marketplace list --search ${ref}\n` + +' then pass its id with --dataset-id ' + ); + return undefined; +}; + +const parse_records_limit = (raw: string|undefined): number|undefined=>{ + // Required, unlike the API, which treats it as optional. Omitting it means + // "no cap" on datasets holding hundreds of millions of records, and cost + // is only observable after the query is committed — so there is no way to + // undo an accidentally huge query, and no way to preview its price. + if (raw === undefined) + { + fail( + '--records-limit is required.\n' + +' Marketplace queries are billed by records returned, and some ' + +'datasets\n' + +' hold hundreds of millions. Pass an explicit cap, e.g. ' + +'--records-limit 1000.' + ); + return undefined; + } + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) + { + fail( + `Invalid --records-limit "${raw}".\n` + +' Use a positive integer.' + ); + return undefined; + } + return value; +}; + +const load_filter = (opts: Marketplace_opts): unknown|undefined=>{ + if (opts.filter !== undefined && opts.filterFile !== undefined) + { + fail('Provide either --filter or --filter-file, not both.'); + return undefined; + } + let text: string; + if (opts.filterFile !== undefined) + { + try { + text = fs.readFileSync(opts.filterFile, 'utf8'); + } catch(e) { + fail( + `Cannot read filter file "${opts.filterFile}": ` + +`${(e as Error).message}` + ); + return undefined; + } + } + else if (opts.filter !== undefined) + text = opts.filter; + else + { + fail( + 'No filter provided.\n' + +' Pass --filter \'\' or --filter-file .\n' + +' Example: --filter \'{"name":"industry","operator":"=",' + +'"value":"Technology"}\'' + ); + return undefined; + } + const trimmed = text.trim(); + if (!trimmed) + { + fail('Filter is empty.'); + return undefined; + } + try { + return JSON.parse(trimmed) as unknown; + } catch(e) { + fail( + `Invalid JSON in filter: ${(e as Error).message}\n` + +' The filter is a JSON object, e.g. ' + +'{"name":"followers","operator":">","value":10000}' + ); + return undefined; + } +}; + +const DESCRIPTION_WIDTH = 68; + +const truncate = (text: string|undefined): string=>{ + const clean = (text ?? '').replace(/\s+/g, ' ').trim(); + if (clean.length <= DESCRIPTION_WIDTH) + return clean; + return clean.slice(0, DESCRIPTION_WIDTH-1)+'…'; +}; + +const NO_RECORDS_CODE = 'no_records_found'; + +// The API reports a zero-match query as status:failed with warning_code +// no_records_found. That is a successful query with an empty result, not a +// failure, and it is reported as such (exit 0) — consistent with how +// `marketplace list --search` reports finding nothing. +const is_empty_result = (snap: Snapshot_status): boolean=> + snap.warning_code == NO_RECORDS_CODE; + +const snapshot_error = (snap: Snapshot_status): string=> + snap.error || snap.error_message || snap.failure_reason || snap.warning + || snap.message || 'no reason returned by the API'; + +const handle_list = async(opts: Marketplace_opts)=>{ + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner('Fetching dataset catalogue...'); + try { + const all = await get(api_key, LIST_ENDPOINT, + {timing: opts.timing}); + spinner.stop(); + let rows = Array.isArray(all) ? all : []; + if (opts.featured) + { + const ids = new Set(Object.values(FEATURED_DATASETS)); + rows = rows.filter(d=>ids.has(d.id)); + } + if (opts.search) + { + const needle = opts.search.toLowerCase(); + rows = rows.filter(d=> + (d.name ?? '').toLowerCase().includes(needle) + || d.id.toLowerCase().includes(needle)); + } + if (!rows.length) + { + info(opts.search + ? `No datasets match "${opts.search}".` + : 'No datasets returned.'); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(rows, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + // Names carry stray whitespace in the catalogue; trim for display only. + print_table(rows.map(d=>({ + name: (d.name ?? '').trim(), + id: d.id, + records: d.size === undefined ? '' : d.size.toLocaleString(), + })), ['name', 'id', 'records']); + if (rows.length > LIST_WARN_THRESHOLD) + { + info(`${rows.length} datasets listed. Narrow with --search , ` + +'or use --featured for the named shortcuts.'); + } + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const handle_fields = async(dataset: string, opts: Marketplace_opts)=>{ + const dataset_id = resolve_dataset_id({...opts, dataset}); + if (!dataset_id) + return; + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner(`Fetching fields for ${dataset_id}...`); + try { + const meta = await get( + api_key, metadata_endpoint(dataset_id), {timing: opts.timing}); + spinner.stop(); + const fields = meta?.fields ?? {}; + const names = Object.keys(fields); + if (!names.length) + { + info(`No fields reported for ${dataset_id}.`); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(meta, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + print_table(names.sort().map(name=>({ + field: name, + type: fields[name]?.type ?? '', + required: fields[name]?.required ? 'yes' : '', + // Descriptions run to several hundred characters; the table is for + // scanning. --json gives the untruncated text. + description: truncate(fields[name]?.description), + })), ['field', 'type', 'required', 'description']); + info(`${names.length} fields — any of them can be used in a filter.`); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const fetch_snapshot_status = ( + api_key: string, + snapshot_id: string, + timing: boolean|undefined +): Promise=> + get(api_key, snapshot_endpoint(snapshot_id), {timing}); + +const print_snapshot_summary = (snap: Snapshot_status)=>{ + const bits: string[] = []; + if (snap.dataset_size !== undefined) + bits.push(`${snap.dataset_size.toLocaleString()} records`); + if (snap.file_size !== undefined) + bits.push(`${(snap.file_size/1024/1024).toFixed(2)} MB`); + if (snap.cost !== undefined) + bits.push(`cost ${snap.cost}`); + if (bits.length) + info(bits.join(' · ')); +}; + +// Waits for a snapshot to leave scheduled/building, then hands back the final +// status. Note the 1s cadence: poll_until counts attempts rather than elapsed +// time, so timeout_seconds only means seconds while the interval stays 1s. +// (PR #20 upstream converts that loop to a wall-clock deadline; once it lands, +// this can move to the SDK's gentler 5s pacing without changing --timeout.) +const wait_for_snapshot = async( + api_key: string, + snapshot_id: string, + timeout: number, + opts: Marketplace_opts +): Promise=>{ + const poll_result = await poll_until({ + timeout_seconds: timeout, + fetch_once: ()=>fetch_snapshot_status(api_key, snapshot_id, + opts.timing), + get_status: snap=>snap.status, + running_statuses: RUNNING_STATUSES, + timeout_label: `snapshot ${snapshot_id}`, + on_running: ({attempt, timeout_seconds, status})=>{ + console.error(dim( + `Status: ${status} - polling again ` + +`(attempt ${attempt}/${timeout_seconds})` + )); + }, + }); + return poll_result.result; +}; + +const download_snapshot = async( + api_key: string, + snapshot_id: string, + format: Marketplace_format, + opts: Marketplace_opts +)=>{ + const endpoint = `${download_endpoint(snapshot_id)}?format=${format}`; + // csv/jsonl must reach the user byte-for-byte. They cannot go through the + // client's JSON path: it selects it with content_type.includes( + // 'application/json'), which is also true for the 'application/jsonl' this + // endpoint returns — so a JSONL body would be parsed into an object and + // re-serialised as indented JSON. raw_buffer bypasses that entirely. + const data = format == 'json' + ? await get(api_key, endpoint, {timing: opts.timing}) + : (await get(api_key, endpoint, + {timing: opts.timing, raw_buffer: true})).toString('utf8'); + // An empty result is a normal outcome of a filter query and must be said + // out loud — otherwise it is indistinguishable from a silent failure. + const empty = data === undefined || data === null || data === '' + || (Array.isArray(data) && !data.length); + if (empty) + { + info('No records in this snapshot (the filter matched nothing).'); + return; + } + print(data, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); +}; + +const handle_status = async(snapshot_id: string, opts: Marketplace_opts)=>{ + const api_key = ensure_authenticated(opts.apiKey); + const spinner = start_spinner(`Checking snapshot "${snapshot_id}"...`); + let snap: Snapshot_status; + // The try covers only the request: a failed *snapshot* is a verdict to + // report, not an exception to swallow. + try { + snap = await fetch_snapshot_status(api_key, snapshot_id, opts.timing); + spinner.stop(); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + if (opts.json || opts.pretty || opts.output) + { + print(snap, { + json: opts.json, + pretty: opts.pretty, + output: opts.output, + }); + return; + } + info(`Status: ${snap.status ?? 'unknown'}`); + print_snapshot_summary(snap); + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records.'); + return; + } + if (snap.status == FAILED_STATUS) + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); +}; + +const handle_download = async(snapshot_id: string, opts: Marketplace_opts)=>{ + const format = resolve_format(opts.format); + const api_key = ensure_authenticated(opts.apiKey); + let timeout = 600; + try { + timeout = parse_timeout(opts.timeout); + } catch(e) { + fail((e as Error).message); + return; + } + const spinner = start_spinner(`Fetching snapshot "${snapshot_id}"...`); + let snap: Snapshot_status; + try { + snap = await fetch_snapshot_status(api_key, snapshot_id, opts.timing); + spinner.stop(); + const running = !!snap.status && RUNNING_STATUSES.includes(snap.status); + if (running && !opts.wait) + { + info(`Snapshot "${snapshot_id}" is not ready yet ` + +`(status: ${snap.status}).`); + info('Re-run with --wait, or check it with: ' + +`brightdata marketplace status ${snapshot_id}`); + // 3 = not ready (retryable), distinct from 1 = error. + process.exit(EXIT.NOT_READY); + return; + } + if (running) + snap = await wait_for_snapshot(api_key, snapshot_id, timeout, opts); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + // A failed snapshot is a verdict, reported outside the request try/catch. + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records — nothing to download.'); + return; + } + if (snap.status == FAILED_STATUS) + { + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); + return; + } + print_snapshot_summary(snap); + try { + await download_snapshot(api_key, snapshot_id, format, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const handle_filter = async(opts: Marketplace_opts)=>{ + const dataset_id = resolve_dataset_id(opts); + if (!dataset_id) + return; + const filter = load_filter(opts); + if (filter === undefined) + return; + const records_limit = parse_records_limit(opts.recordsLimit); + if (records_limit === undefined) + return; + const format = resolve_format(opts.format); + const api_key = ensure_authenticated(opts.apiKey); + let timeout = 600; + try { + timeout = parse_timeout(opts.timeout); + } catch(e) { + fail((e as Error).message); + return; + } + const body: Filter_request = {dataset_id, filter, records_limit}; + const spinner = start_spinner(`Filtering ${dataset_id}...`); + // Each try wraps one request only; verdicts about the result are reported + // after it, so the handler's own catch cannot swallow them. + let res: Filter_response; + try { + res = await post(api_key, FILTER_ENDPOINT, body, + {timing: opts.timing}); + spinner.stop(); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + const snapshot_id = res?.snapshot_id; + if (!snapshot_id) + { + fail( + 'Failed to create snapshot: ' + +(res?.error || res?.message || res?.failure_reason + || 'no snapshot_id in response') + ); + return; + } + if (opts.async) + { + success(`Filter submitted. Snapshot ID: ${snapshot_id}`); + info(`Check it with: brightdata marketplace status ` + +`${snapshot_id}`); + info('Download when ready: brightdata marketplace download ' + +`${snapshot_id} --wait`); + return; + } + console.error(dim(`Created snapshot ${snapshot_id}`)); + let snap: Snapshot_status; + try { + snap = await wait_for_snapshot(api_key, snapshot_id, timeout, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + return; + } + if (snap.status == FAILED_STATUS && is_empty_result(snap)) + { + info('The filter matched no records.'); + return; + } + if (snap.status == FAILED_STATUS) + { + fail(`Snapshot ${snapshot_id} failed: ${snapshot_error(snap)}`); + return; + } + print_snapshot_summary(snap); + try { + await download_snapshot(api_key, snapshot_id, format, opts); + } catch(e) { + console.error((e as Error).message); + process.exit(EXIT.ERROR); + } +}; + +const list_command = new Command('list') + .description('List datasets in the marketplace catalogue') + .option('--featured', 'Only the datasets with named shortcuts') + .option('--search ', 'Filter by text in the name or id') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_list); + +const fields_command = new Command('fields') + .description('Show a dataset\'s filterable fields and their types') + .argument('', 'Dataset name (see list --featured) or gd_ id') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_fields); + +const filter_command = new Command('filter') + .description('Query a dataset and download the matching records') + .option('--dataset ', + 'Dataset name (see \'marketplace list --featured\')') + .option('--dataset-id ', 'Raw dataset id (gd_...), for any dataset') + .option('--filter ', 'Filter tree as inline JSON') + .option('--filter-file ', 'Filter tree from a JSON file') + .option('--records-limit ', + 'Maximum records to return (REQUIRED — queries are billed ' + +'by records)') + .option('--format ', 'Result format: json, jsonl, csv (default: json)') + .option('--timeout ', + 'Polling timeout in seconds ' + +'(default: 600 or BRIGHTDATA_POLLING_TIMEOUT)') + .option('--async', 'Submit only; print the snapshot ID and exit') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_filter); + +const status_command = new Command('status') + .description('Check a marketplace snapshot (status, records, size, cost)') + .argument('', 'Snapshot ID returned by marketplace filter') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_status); + +const download_command = new Command('download') + .description( + 'Download a marketplace snapshot ' + +'(exit codes: 0 data, 1 error, 3 not ready yet)' + ) + .argument('', 'Snapshot ID returned by marketplace filter') + .option('--wait', 'Poll until the snapshot is ready, then download') + .option('--format ', 'Result format: json, jsonl, csv (default: json)') + .option('--timeout ', + 'Polling timeout in seconds with --wait ' + +'(default: 600 or BRIGHTDATA_POLLING_TIMEOUT)') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .action(handle_download); + +const marketplace_command = new Command('marketplace') + .description( + 'Query Bright Data\'s pre-collected datasets ' + +'(pipelines collects new data; this queries data that already exists)' + ) + .addCommand(list_command) + .addCommand(fields_command) + .addCommand(filter_command) + .addCommand(status_command) + .addCommand(download_command); + +add_examples(marketplace_command, [ + { + description: 'See the datasets that have named shortcuts', + command: 'brightdata marketplace list --featured', + }, + { + description: 'Find any other dataset in the catalogue', + command: 'brightdata marketplace list --search zillow', + }, + { + description: 'Inspect what you can filter on before spending anything', + command: 'brightdata marketplace fields linkedin_people_profiles', + }, + { + description: 'Query a dataset (a record cap is always required)', + command: 'brightdata marketplace filter ' + +'--dataset linkedin_company_profiles ' + +'--filter \'{"name":"industry","operator":"=",' + +'"value":"Technology"}\' --records-limit 1000 ' + +'--format jsonl -o tech.jsonl', + }, + { + description: 'Submit now, collect later', + command: 'brightdata marketplace filter --dataset x_twitter_posts ' + +'--filter \'{"name":"likes","operator":">","value":10000}\' ' + +'--records-limit 500 --async', + }, +]); + +export {marketplace_command, handle_list, handle_fields, handle_filter, + handle_status, handle_download, resolve_dataset_id, FEATURED_DATASETS}; diff --git a/src/index.ts b/src/index.ts index 2136762..4ea6912 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import {logout_command} from './commands/logout'; import {scrape_command} from './commands/scrape'; import {search_command} from './commands/search'; import {pipelines_command} from './commands/dataset'; +import {marketplace_command} from './commands/marketplace'; import {status_command} from './commands/status'; import {zones_command} from './commands/zones'; import {config_command} from './commands/config'; @@ -41,6 +42,7 @@ const build_program = ()=>{ program.addCommand(scrape_command); program.addCommand(search_command); program.addCommand(pipelines_command); + program.addCommand(marketplace_command); program.addCommand(status_command); program.addCommand(zones_command); program.addCommand(config_command); diff --git a/src/types/marketplace.ts b/src/types/marketplace.ts new file mode 100644 index 0000000..fcada1d --- /dev/null +++ b/src/types/marketplace.ts @@ -0,0 +1,87 @@ +// Dataset Marketplace: querying pre-collected records. Distinct from the +// /datasets/v3/* Web Scraper API that `pipelines` uses — that one collects new +// data and bills for scraping; this one filters data Bright Data already holds. +// Note the formats differ too: no ndjson here. +type Marketplace_format = 'json'|'jsonl'|'csv'; + +type Dataset_info = { + id: string; + name?: string; + size?: number; // record count +}; + +type Dataset_field = { + type?: string; // text | number | url | array | object | boolean + active?: boolean; + required?: boolean; + description?: string; +}; + +type Dataset_metadata = { + id?: string; + fields?: Record; +}; + +type Snapshot_status = { + id?: string; + snapshot_id?: string; + status?: 'scheduled'|'building'|'ready'|'failed'|string; + dataset_id?: string; + dataset_size?: number; // records + file_size?: number; // bytes + cost?: number; + error?: string; + error_message?: string; + failure_reason?: string; + message?: string; + // A zero-match query comes back as status:failed with these set, rather + // than as a ready snapshot holding no rows. + warning?: string; + warning_code?: string; +}; + +// records_limit is optional in the API but REQUIRED by this CLI: omitting it +// means "no cap" against datasets holding hundreds of millions of records, and +// cost is only observable after the query is committed. +type Filter_request = { + dataset_id: string; + filter: unknown; + records_limit: number; +}; + +type Filter_response = { + snapshot_id?: string; + error?: string; + message?: string; + failure_reason?: string; +}; + +type Marketplace_opts = { + dataset?: string; + datasetId?: string; + filter?: string; + filterFile?: string; + recordsLimit?: string; + featured?: boolean; + search?: string; + format?: string; + timeout?: string; + wait?: boolean; + async?: boolean; + output?: string; + json?: boolean; + pretty?: boolean; + timing?: boolean; + apiKey?: string; +}; + +export type { + Marketplace_format, + Dataset_info, + Dataset_field, + Dataset_metadata, + Snapshot_status, + Filter_request, + Filter_response, + Marketplace_opts, +}; diff --git a/src/utils/exit-codes.ts b/src/utils/exit-codes.ts new file mode 100644 index 0000000..cdf0524 --- /dev/null +++ b/src/utils/exit-codes.ts @@ -0,0 +1,9 @@ +// Exit codes are cross-command public API for shell users. +// 0 success · 1 error (fail() and catch-all paths) · 3 result not ready yet +const EXIT = { + OK: 0, + ERROR: 1, + NOT_READY: 3, +} as const; + +export {EXIT};