Product Sourcing Tutorial

ssh root@115.29.238.14

Expected output:

root@115.29.238.14's password: ********
Welcome to Ubuntu 20.04 LTS (GNU/Linux 5.4.0-generic x86_64)

root@iZbp16bd840bjqfvwjnqxeZ:~#

apt update && apt upgrade -y

Expected output:

Hit:1 http://mirrors.cloud.aliyuncs.com/ubuntu focal InRelease
Hit:2 http://mirrors.cloud.aliyuncs.com/ubuntu focal-updates InRelease
Reading package lists... Done
Building dependency tree
Reading state information... Done
...
Processing triggers for libc-bin (2.31-0ubuntu9.18) ...
Processing triggers for systemd (245.4-4ubuntu3.18) ...

curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt install -y nodejs

# Verify
node -v   # should show v20.x.x
npm -v    # should show 10.x.x

Expected output:

## Installing the NodeSource Node.js 20.x repo...
...
Reading package lists... Done
Setting up nodejs (20.x-1nodesource1) ...

root@iZbp16bd840bjqfvwjnqxeZ:~# node -v
v20.x.x
root@iZbp16bd840bjqfvwjnqxeZ:~# npm -v
10.x.x

# Install system dependencies for Chromium
apt install -y libnss3 libatk1.0-0 libatk-bridge2.0-0 \
  libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 \
  libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2

# Install Playwright globally
npm install -g playwright

# Download Chromium browser
npx playwright install chromium

# Test it works
node -e "const {chromium}=require('playwright'); (async()=>{const b=await chromium.launch();console.log('Browser works!');await b.close();})()"

Expected output:

Reading package lists... Done
libnss3 is already the newest version (3.x-1)
...
added 1 package in 2s

root@iZbp16bd840bjqfvwjnqxeZ:~# npx playwright install chromium
Downloading Chromium 1xx.x.x (playwright build) from
https://playwright.azureedge.net/builds/...
Chromium 1xx.x.x downloaded

root@iZbp16bd840bjqfvwjnqxeZ:~# node -e "..."
Browser works!

npm install -g pm2

Expected output:

added 1 package in 3s
root@iZbp16bd840bjqfvwjnqxeZ:~#

mkdir -p /root/scraper-api
cd /root/scraper-api
npm init -y
npm install express cors playwright

Expected output:

root@iZbp16bd840bjqfvwjnqxeZ:~# mkdir -p /root/scraper-api
root@iZbp16bd840bjqfvwjnqxeZ:~# cd /root/scraper-api
root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# npm init -y
Wrote to /root/scraper-api/package.json:

{
  "name": "scraper-api",
  "version": "1.0.0",
  "main": "server.js",
  ...
}

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# npm install express cors playwright
added 80 packages, and audited 81 packages in 10s
found 0 vulnerabilities

// Save as: /root/scraper-api/server.js

const express = require("express");
const cors = require("cors");
const { chromium } = require("playwright");

const app = express();
app.use(cors());
app.use(express.json());

// API key security — only your Base44 app can call this
const API_KEY = process.env.SCRAPER_API_KEY || "bd2china-secret-2026";

// ─── Taobao Search ───
app.post("/api/search/taobao", async (req, res) => {
  const { keyword, api_key } = req.body;
  if (api_key !== API_KEY) return res.status(401).json({ error: "Unauthorized" });

  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  try {
    const url = "https://s.taobao.com/search?q=" + encodeURIComponent(keyword);
    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
    await page.waitForTimeout(3000);

    const products = await page.$$eval("[class*=Content]", items => {
      return items.slice(0, 10).map(item => {
        const titleEl = item.querySelector("[class*=Title] span, a[title]");
        const priceEl = item.querySelector("[class*=Price]");
        const linkEl = item.querySelector("a[href]");
        const imgEl = item.querySelector("img[src]");
        return {
          title: (titleEl && titleEl.textContent && titleEl.textContent.trim()) || "",
          price: (priceEl && priceEl.textContent && priceEl.textContent.trim()) || "",
          link: linkEl ? linkEl.href : "",
          image: imgEl ? imgEl.src : "",
          platform: "taobao"
        };
      }).filter(p => p.title);
    });

    res.json({ success: true, keyword, count: products.length, products });
  } catch (error) {
    res.json({ success: false, error: error.message, products: [] });
  } finally {
    await browser.close();
  }
});

// ─── Pinduoduo Search ───
app.post("/api/search/pinduoduo", async (req, res) => {
  const { keyword, api_key } = req.body;
  if (api_key !== API_KEY) return res.status(401).json({ error: "Unauthorized" });

  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  try {
    const url = "https://mobile.yangkeduo.com/proxy/api/search?q=" + encodeURIComponent(keyword);
    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
    await page.waitForTimeout(3000);
    const products = await page.$$eval("div[class*=goods]", items => {
      return items.slice(0, 10).map(item => ({
        title: (item.querySelector("[class*=title]") || {}).textContent || "",
        price: (item.querySelector("[class*=price]") || {}).textContent || "",
        image: item.querySelector("img") ? item.querySelector("img").src : "",
        link: item.querySelector("a") ? item.querySelector("a").href : "",
        platform: "pinduoduo"
      })).filter(p => p.title);
    });
    res.json({ success: true, keyword, count: products.length, products });
  } catch (error) {
    res.json({ success: false, error: error.message, products: [] });
  } finally {
    await browser.close();
  }
});

// ─── 1688 (Alibaba) Search ───
app.post("/api/search/1688", async (req, res) => {
  const { keyword, api_key } = req.body;
  if (api_key !== API_KEY) return res.status(401).json({ error: "Unauthorized" });

  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  try {
    const url = "https://s.1688.com/selloffer/offer_search.htm?keywords=" + encodeURIComponent(keyword);
    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
    await page.waitForTimeout(3000);
    const products = await page.$$eval("[class*=offer]", items => {
      return items.slice(0, 10).map(item => ({
        title: (item.querySelector("[class*=title], .title") || {}).textContent || "",
        price: (item.querySelector("[class*=price], .price") || {}).textContent || "",
        image: item.querySelector("img") ? item.querySelector("img").src : "",
        link: item.querySelector("a") ? item.querySelector("a").href : "",
        platform: "1688"
      })).filter(p => p.title);
    });
    res.json({ success: true, keyword, count: products.length, products });
  } catch (error) {
    res.json({ success: false, error: error.message, products: [] });
  } finally {
    await browser.close();
  }
});

// ─── Baidu Search (general product info) ───
app.post("/api/search/baidu", async (req, res) => {
  const { keyword, api_key } = req.body;
  if (api_key !== API_KEY) return res.status(401).json({ error: "Unauthorized" });

  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  try {
    const url = "https://www.baidu.com/s?wd=" + encodeURIComponent(keyword);
    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 20000 });
    const results = await page.$$eval(".result", items => {
      return items.slice(0, 10).map(item => ({
        title: (item.querySelector("h3 a") || {}).textContent || "",
        link: item.querySelector("h3 a") ? item.querySelector("h3 a").href : "",
        snippet: (item.querySelector("[class*=content], .c-abstract") || {}).textContent || "",
        platform: "baidu"
      })).filter(r => r.title);
    });
    res.json({ success: true, keyword, count: results.length, products: results });
  } catch (error) {
    res.json({ success: false, error: error.message, products: [] });
  } finally {
    await browser.close();
  }
});

// ─── Search ALL platforms ───
app.post("/api/search/all", async (req, res) => {
  const { keyword, api_key } = req.body;
  if (api_key !== API_KEY) return res.status(401).json({ error: "Unauthorized" });

  const platforms = ["taobao", "pinduoduo", "1688", "baidu"];
  const allResults = {};

  for (const platform of platforms) {
    try {
      const resp = await fetch("http://localhost:3000/api/search/" + platform, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ keyword, api_key })
      });
      allResults[platform] = await resp.json();
    } catch (e) {
      allResults[platform] = { success: false, error: e.message, products: [] };
    }
  }

  res.json({ success: true, keyword, results: allResults });
});

// ─── Health check ───
app.get("/api/health", (req, res) => {
  res.json({ status: "ok", server: "bd2china-scraper", time: new Date().toISOString() });
});

const PORT = 3000;
app.listen(PORT, "0.0.0.0", () => {
  console.log("Scraper API running on port " + PORT);
});

cd /root/scraper-api

# Set API key as environment variable
export SCRAPER_API_KEY="bd2china-secret-2026"

# Start with PM2 (auto-restarts on crash)
pm2 start server.js --name scraper-api

# Save PM2 config (auto-start on reboot)
pm2 save
pm2 startup

# Check it is running
pm2 status
curl http://localhost:3000/api/health

Expected output:

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# pm2 start server.js --name scraper-api
[PM2] Starting /root/scraper-api/server.js in fork_mode (1 instance)
[PM2] Done.
+---+------------------+----------+---------+----------+
| id| name             | mode     | status  | restarts |
+---+------------------+----------+---------+----------+
| 0 | scraper-api      | fork     | online  | 0        |
+---+------------------+----------+---------+----------+

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# pm2 save
[PM2] Saving current process list...
[PM2] Successfully saved in /root/.pm2/dump.pm2

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# pm2 startup
[PM2] Init System found: systemd
...
[PM2] [v] Command successfully executed.

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# pm2 status
+---+------------------+-----------+
| id| name             | status    |
+---+------------------+-----------+
| 0 | scraper-api      | online    |
+---+------------------+-----------+

root@iZbp16bd840bjqfvwjnqxeZ:/root/scraper-api# curl http://localhost:3000/api/health
{"status":"ok","server":"bd2china-scraper","time":"2026-07-08T..."}