How to use async jobs and webhooks for bulk enrichment

For enriching more than 50 records at a time, the synchronous endpoints are not the right tool. This guide explains how to use ApiOne's async job system to submit large batches, poll for status, and receive results via webhook — without blocking your application or hitting rate limits.

When to use async jobs

Step 1 — Submit an async job

async function submitAsyncJob(domains, webhookUrl) {
  const response = await fetch('https://apione.store/api/v1/async/enrich', {
    method: 'POST',
    headers: {
      'X-API-Key': process.env.APIONE_API_KEY,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      type: 'company',           // 'company', 'people', 'email', 'tech', or 'signals'
      records: domains.map(domain => ({ domain })),
      webhook_url: webhookUrl,   // Optional — omit to poll instead
    }),
  });

  const result = await response.json();
  console.log('Job submitted:', result.job_id);
  console.log('Estimated completion:', result.estimated_completion);
  return result.job_id;
}

Step 2 — Poll for job status (alternative to webhook)

async function pollJobStatus(jobId) {
  const response = await fetch(`https://apione.store/api/v1/async/status/${jobId}`, {
    headers: { 'X-API-Key': process.env.APIONE_API_KEY },
  });
  return await response.json();
}

async function waitForJob(jobId, intervalMs = 5000, maxWaitMs = 300000) {
  const start = Date.now();

  while (Date.now() - start < maxWaitMs) {
    const status = await pollJobStatus(jobId);

    if (status.status === 'completed') {
      console.log(`Job complete: ${status.results_count} results`);
      return status.results;
    }

    if (status.status === 'failed') {
      throw new Error(`Job failed: ${status.error}`);
    }

    console.log(`Job ${status.status}: ${status.progress}% complete`);
    await new Promise(r => setTimeout(r, intervalMs));
  }

  throw new Error('Job timed out');
}

Step 3 — Receive results via webhook

Webhooks are the recommended approach for production use. Your server receives a POST request when the job completes:

import express from 'express';
import crypto from 'crypto';

const app = express();
app.use(express.json());

app.post('/webhooks/apione', (req, res) => {
  // 1. Verify the webhook signature
  const signature = req.headers['x-apione-signature'];
  const expectedSig = crypto
    .createHmac('sha256', process.env.APIONE_WEBHOOK_SECRET)
    .update(JSON.stringify(req.body))
    .digest('hex');

  if (signature !== `sha256=${expectedSig}`) {
    return res.status(401).send('Invalid signature');
  }

  // 2. Acknowledge immediately (must respond within 5 seconds)
  res.sendStatus(200);

  // 3. Process results asynchronously
  processWebhookResults(req.body).catch(console.error);
});

async function processWebhookResults({ event, job_id, results }) {
  if (event !== 'job.completed') return;

  console.log(`Processing ${results.length} results for job ${job_id}`);

  for (const record of results) {
    if (record.status === 'success') {
      await saveEnrichedData(record.input, record.data);
    } else if (record.status === 'not_found') {
      await markAsNotFound(record.input);
    }
  }
}

Webhook payload structure

{
  "event": "job.completed",
  "job_id": "job_abc123",
  "completed_at": "2024-11-15T14:32:00Z",
  "total_records": 500,
  "successful": 423,
  "not_found": 67,
  "failed": 10,
  "credits_used": 2182,
  "results": [
    {
      "status": "success",
      "input": { "domain": "stripe.com" },
      "data": { "company_name": "Stripe", "industry": "Financial Technology", ... }
    },
    {
      "status": "not_found",
      "input": { "domain": "unknownco.io" },
      "data": null
    }
  ]
}

Retry logic for webhook failures

ApiOne retries failed webhook deliveries up to 3 times with exponential backoff (1min, 5min, 30min). To avoid duplicate processing, store the job_id and check for it before processing:

async function processWebhookResults({ event, job_id, results }) {
  if (event !== 'job.completed') return;

  // Idempotency check
  const alreadyProcessed = await db.processedJobs.findOne({ job_id });
  if (alreadyProcessed) {
    console.log('Duplicate webhook — skipping:', job_id);
    return;
  }

  // Mark as processed before processing (prevents double-processing on crash)
  await db.processedJobs.insert({ job_id, processed_at: new Date() });

  for (const record of results) {
    if (record.status === 'success') {
      await saveEnrichedData(record.input, record.data);
    }
  }
}

Related

• Async Jobs API reference
• Webhooks reference
• Guide: CRM enrichment at scale