scrape-rakuten-books.ts

import axios from "axios";
import lowdb from "lowdb";
import FileSync from "lowdb/adapters/FileSync";
import genres from "./genres.json";

const db = lowdb(new FileSync("db.json"));
const http = axios.create({
  baseURL: "https://app.rakuten.co.jp/services/api",
  validateStatus: () => true,
  params: {
    applicationId: "",
    affiliateId: "",
    format: "json",
    formatVersion: 2,
    genrePath: 1,
  },
});

const useGenres = [
  "漫画（コミック）",
  "人文・思想・社会",
  "美容・暮らし・健康・料理",
  "エンタメ・ゲーム",
  "ホビー・スポーツ・美術",
  "パソコン・システム開発",
  "ビジネス・経済・就職",
  "科学・技術",
].map((name) => genres.filter((g) => g.booksGenreName === name)[0]);

const getBooks = async (params: any) => {
  const resp = await http.get("/BooksBook/Search/20170404", { params });
  return resp.data.Items;
};

function sleep(wait: number) {
  return new Promise(function (resolve) {
    setTimeout(function () {
      resolve();
    }, wait);
  });
}

(async () => {
  // const resp = await http.get("/BooksGenre/Search/20121128", {
  //   params: {
  //     booksGenreId: "001011",
  //   },
  // });
  // console.log(JSON.stringify(resp.data));
  // const resp = await http.get("/BooksBook/Search/20170404", {
  //   params: {
  //     booksGenreId: "001008",
  //     sort: "reviewCount",
  //     page: 100,
  //   },
  // });
  // console.log(resp.data);
  for await (const genre of useGenres) {
    let books: any = [];
    for (let page = 1; page <= 100; page++) {
      const got = await getBooks({
        booksGenreId: genre.booksGenreId,
        sort: "reviewCount",
        page,
      });
      books = books.concat(got);
      await sleep(1000);
      console.log(`booksGenre=${genre.booksGenreName}/page=${page}`);
    }
    db.set(`G_${genre.booksGenreId}`, books).write();
  }
})();