Varun Shah
e32d475aa9
- Add authentication scripts with SubtleCrypto password encryption - Add sourcemap extraction pipeline (update-urls, download-sourcemaps, extract-sources) - Add Playwright API interception script for monetization endpoints - Document two-step auth flow with JWT tokens and dual cookies - Move extracted source from root to anyclip/ directory - Add project configuration (.env.example, .gitignore, CLAUDE.md)
67 lines
2.2 KiB
TypeScript
67 lines
2.2 KiB
TypeScript
#!/usr/bin/env bun
|
|
/**
|
|
* Fetch all JS URLs from AnyClip's build manifest and pages, write to urls.txt
|
|
*/
|
|
|
|
const BASE_URL = "https://videomanager.anyclip.com";
|
|
const BUILD_MANIFEST_URL =
|
|
"https://videomanager.anyclip.com/_next/static/pBwdVlmlgRKGbPM_149Sv/_buildManifest.js";
|
|
|
|
// Pages to scrape for additional JS files
|
|
const PAGES_TO_SCRAPE = [
|
|
"https://videomanager.anyclip.com/login",
|
|
];
|
|
|
|
const jsUrls = new Set<string>();
|
|
|
|
// 1. Fetch build manifest
|
|
console.log("Fetching build manifest...");
|
|
const manifestResponse = await fetch(BUILD_MANIFEST_URL);
|
|
const manifestText = await manifestResponse.text();
|
|
|
|
// Extract all JS chunk paths from manifest
|
|
const manifestChunks = [...manifestText.matchAll(/static\/chunks\/[^"'\s,\)]+\.js/g)]
|
|
.map((m) => `${BASE_URL}/_next/${m[0]}`);
|
|
manifestChunks.forEach((url) => jsUrls.add(url));
|
|
console.log(` Found ${manifestChunks.length} chunks in manifest`);
|
|
|
|
// 2. Scrape pages for additional JS files
|
|
for (const pageUrl of PAGES_TO_SCRAPE) {
|
|
console.log(`Scraping ${pageUrl}...`);
|
|
try {
|
|
const response = await fetch(pageUrl);
|
|
const html = await response.text();
|
|
|
|
// Extract JS URLs from page
|
|
const pageJsUrls = [
|
|
...html.matchAll(/src="(\/_next\/static\/[^"]+\.js)"/g),
|
|
].map((m) => `${BASE_URL}${m[1]}`);
|
|
|
|
pageJsUrls.forEach((url) => jsUrls.add(url));
|
|
console.log(` Found ${pageJsUrls.length} JS files`);
|
|
} catch (e) {
|
|
console.log(` Failed to scrape: ${e}`);
|
|
}
|
|
}
|
|
|
|
// 3. Read existing urls.txt and merge
|
|
const existingUrls = new Set<string>();
|
|
const urlsFile = Bun.file("urls.txt");
|
|
if (await urlsFile.exists()) {
|
|
const content = await urlsFile.text();
|
|
content.split("\n").filter(Boolean).forEach((url) => existingUrls.add(url.trim()));
|
|
console.log(`\nExisting urls.txt has ${existingUrls.size} URLs`);
|
|
}
|
|
|
|
// Merge
|
|
const allUrls = new Set([...existingUrls, ...jsUrls]);
|
|
const newUrls = [...jsUrls].filter((url) => !existingUrls.has(url));
|
|
|
|
// 4. Write to urls.txt
|
|
const sortedUrls = [...allUrls].sort();
|
|
await Bun.write("urls.txt", sortedUrls.join("\n") + "\n");
|
|
|
|
console.log(`\n✅ urls.txt updated`);
|
|
console.log(` Total URLs: ${allUrls.size}`);
|
|
console.log(` New URLs added: ${newUrls.length}`);
|