Skip to content

Instantly share code, notes, and snippets.

@euberdeveloper
Last active February 5, 2022 23:42
Show Gist options
  • Save euberdeveloper/5e2aafc0768834d2089eb88a5661738a to your computer and use it in GitHub Desktop.
Save euberdeveloper/5e2aafc0768834d2089eb88a5661738a to your computer and use it in GitHub Desktop.
A scraper with Puppeteer that saves all the videos of the course "Program optimization" at TUM Informatik Msc. 2021/2022
#!/bin/bash
for filepath in ./*.webm; do
filename=$(basename -- "$filepath")
filename="${filename%.*}"
ffmpeg -i ${filename}.webm ${filename}.mp4
done
#!/bin/bash
npm init -y
npm i bbb-video-scraper
npm i -g typescript ts-node
import { BBBVideoScraper } from 'bbb-video-scraper';
const BUFFER_SIZE = 4;
async function scrape(dest: string, link: string) {
const scraper = new BBBVideoScraper({
debug: true,
debugScope: dest
});
await scraper.launch();
await scraper.scrape(link, `${dest}.webm`, { delayAfterVideoFinished: 60000 }),
await scraper.close();
}
function suddividiTasks(tasks: [string, string][], buffer: number): [string, string][][] {
return tasks.reduce<[string, string][][]>((acc, task, index) => {
if (index % buffer === 0) {
acc.push([task]);
} else {
acc[acc.length - 1].push(task);
}
return acc;
}, []);
}
async function main() {
const tasks: [string, string][] = [
['2021_10_20', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1634716570637?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1634716570637'],
['2021_10_27', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1635321257537?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1635321257537'],
['2021_10_28', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1635407796201?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1635407796201'],
['2021_11_10', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1636534549579?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1636534549579'],
['2021_11_11', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1636621060646?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1636621060646'],
['2021_11_17', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1637139360824?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1637139360824'],
['2021_11_18', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1637225916108?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1637225916108'],
['2021_11_24', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1637744092635?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1637744092635'],
['2021_11_25', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1637830572495?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1637830572495'],
['2021_12_01', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1638348600511?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1638348600511'],
['2021_12_08', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1638953851969?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1638953851969'],
['2021_12_09', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1639040109723?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1639040109723'],
['2021_12_15', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1639558514916?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1639558514916'],
['2021_12_22', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1640163075025?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1640163075025'],
['2021_12_23', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1640249795107?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1640249795107'],
['2022_01_12', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1641977746168?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1641977746168'],
['2022_01_13', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1642064140802?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1642064140802'],
['2022_01_19', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1642581907417?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1642581907417'],
['2022_01_20', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1642669277883?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1642669277883'],
['2022_01_26', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1643187352966?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1643187352966'],
['2022_01_27', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1643273688023?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1643273688023'],
['2022_02_02', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1643792100283?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1643792100283'],
['2022_02_03', 'https://balancer.bbb.rbg.tum.de/playback/presentation/2.3/f3456b985855f531b992ac267cba49d55e2c7de9-1643878021663?meetingId=f3456b985855f531b992ac267cba49d55e2c7de9-1643878021663'],
]
const bufferedTasks = suddividiTasks(tasks, BUFFER_SIZE);
for (const buffer of bufferedTasks) {
const tasks = buffer.map(([dest, link]) => scrape(dest, link));
await Promise.all(tasks);
}
}
main();
#!/bin/bash
ts-node main.ts
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment