Add AnyClip integration tools and extracted source code
- Add authentication scripts with SubtleCrypto password encryption - Add sourcemap extraction pipeline (update-urls, download-sourcemaps, extract-sources) - Add Playwright API interception script for monetization endpoints - Document two-step auth flow with JWT tokens and dual cookies - Move extracted source from root to anyclip/ directory - Add project configuration (.env.example, .gitignore, CLAUDE.md)
This commit is contained in:
66
scripts/update-urls.ts
Normal file
66
scripts/update-urls.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Fetch all JS URLs from AnyClip's build manifest and pages, write to urls.txt
|
||||
*/
|
||||
|
||||
const BASE_URL = "https://videomanager.anyclip.com";
|
||||
const BUILD_MANIFEST_URL =
|
||||
"https://videomanager.anyclip.com/_next/static/pBwdVlmlgRKGbPM_149Sv/_buildManifest.js";
|
||||
|
||||
// Pages to scrape for additional JS files
|
||||
const PAGES_TO_SCRAPE = [
|
||||
"https://videomanager.anyclip.com/login",
|
||||
];
|
||||
|
||||
const jsUrls = new Set<string>();
|
||||
|
||||
// 1. Fetch build manifest
|
||||
console.log("Fetching build manifest...");
|
||||
const manifestResponse = await fetch(BUILD_MANIFEST_URL);
|
||||
const manifestText = await manifestResponse.text();
|
||||
|
||||
// Extract all JS chunk paths from manifest
|
||||
const manifestChunks = [...manifestText.matchAll(/static\/chunks\/[^"'\s,\)]+\.js/g)]
|
||||
.map((m) => `${BASE_URL}/_next/${m[0]}`);
|
||||
manifestChunks.forEach((url) => jsUrls.add(url));
|
||||
console.log(` Found ${manifestChunks.length} chunks in manifest`);
|
||||
|
||||
// 2. Scrape pages for additional JS files
|
||||
for (const pageUrl of PAGES_TO_SCRAPE) {
|
||||
console.log(`Scraping ${pageUrl}...`);
|
||||
try {
|
||||
const response = await fetch(pageUrl);
|
||||
const html = await response.text();
|
||||
|
||||
// Extract JS URLs from page
|
||||
const pageJsUrls = [
|
||||
...html.matchAll(/src="(\/_next\/static\/[^"]+\.js)"/g),
|
||||
].map((m) => `${BASE_URL}${m[1]}`);
|
||||
|
||||
pageJsUrls.forEach((url) => jsUrls.add(url));
|
||||
console.log(` Found ${pageJsUrls.length} JS files`);
|
||||
} catch (e) {
|
||||
console.log(` Failed to scrape: ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Read existing urls.txt and merge
|
||||
const existingUrls = new Set<string>();
|
||||
const urlsFile = Bun.file("urls.txt");
|
||||
if (await urlsFile.exists()) {
|
||||
const content = await urlsFile.text();
|
||||
content.split("\n").filter(Boolean).forEach((url) => existingUrls.add(url.trim()));
|
||||
console.log(`\nExisting urls.txt has ${existingUrls.size} URLs`);
|
||||
}
|
||||
|
||||
// Merge
|
||||
const allUrls = new Set([...existingUrls, ...jsUrls]);
|
||||
const newUrls = [...jsUrls].filter((url) => !existingUrls.has(url));
|
||||
|
||||
// 4. Write to urls.txt
|
||||
const sortedUrls = [...allUrls].sort();
|
||||
await Bun.write("urls.txt", sortedUrls.join("\n") + "\n");
|
||||
|
||||
console.log(`\n✅ urls.txt updated`);
|
||||
console.log(` Total URLs: ${allUrls.size}`);
|
||||
console.log(` New URLs added: ${newUrls.length}`);
|
||||
Reference in New Issue
Block a user