A tool for parsing traffic on the jetstream and applying a moderation workstream based on regexp based rules

Merge pull request #34 from skywatch-bsky/file-reorganization

File reorganization

authored by

Scarnecchia and committed by
GitHub
f9aa1bd8 a47d27cd

+2558 -143
+45
.github/workflows/ci.yml
··· 1 + name: CI 2 + 3 + on: 4 + push: 5 + branches: [main] 6 + pull_request: 7 + branches: [main] 8 + 9 + jobs: 10 + test: 11 + name: Test 12 + runs-on: ubuntu-latest 13 + 14 + steps: 15 + - name: Checkout code 16 + uses: actions/checkout@v4 17 + 18 + - name: Setup Node.js 19 + uses: actions/setup-node@v4 20 + with: 21 + node-version: '20' 22 + cache: 'npm' 23 + 24 + - name: Install dependencies 25 + run: npm ci 26 + 27 + - name: Run linter 28 + run: npm run lint 29 + 30 + - name: Type check 31 + run: npx tsc --noEmit 32 + 33 + - name: Run tests 34 + run: npm run test:run 35 + 36 + - name: Generate coverage report 37 + run: npm run test:coverage 38 + 39 + - name: Upload coverage to Codecov 40 + uses: codecov/codecov-action@v4 41 + with: 42 + token: ${{ secrets.CODECOV_TOKEN }} 43 + files: ./coverage/coverage-final.json 44 + fail_ci_if_error: false 45 + verbose: true
+1
.gitignore
··· 5 5 labels.db* 6 6 .DS_Store 7 7 src/constants.ts 8 + constants.ts
+1 -1
package.json
··· 1 1 { 2 2 "name": "skywatch-tools", 3 - "version": "1.1.0", 3 + "version": "1.3.0", 4 4 "type": "module", 5 5 "scripts": { 6 6 "start": "npx tsx src/main.ts",
+3 -3
src/checkHandles.ts src/rules/handles/checkHandles.ts
··· 1 1 import { HANDLE_CHECKS } from "./constants.js"; 2 - import { logger } from "./logger.js"; 2 + import { logger } from "../../logger.js"; 3 3 import { 4 4 createAccountReport, 5 5 createAccountComment, 6 6 createAccountLabel, 7 - } from "./moderation.js"; 8 - import { GLOBAL_ALLOW } from "./constants.js"; 7 + } from "../../moderation.js"; 8 + import { GLOBAL_ALLOW } from "../../constants.js"; 9 9 10 10 export const checkHandle = async ( 11 11 did: string,
+38 -14
src/checkPosts.ts src/rules/posts/checkPosts.ts
··· 1 1 import { LINK_SHORTENER, POST_CHECKS } from "./constants.js"; 2 - import { Post } from "./types.js"; 3 - import { logger } from "./logger.js"; 4 - import { countStarterPacks } from "./count.js"; 2 + import { Post } from "../../types.js"; 3 + import { logger } from "../../logger.js"; 4 + import { countStarterPacks } from "../account/countStarterPacks.js"; 5 5 import { 6 6 createPostLabel, 7 7 createAccountReport, 8 8 createAccountComment, 9 9 createPostReport, 10 - } from "./moderation.js"; 11 - import { getFinalUrl, getLanguage } from "./utils.js"; 12 - import { GLOBAL_ALLOW } from "./constants.js"; 10 + } from "../../moderation.js"; 11 + import { getLanguage } from "../../utils/getLanguage.js"; 12 + import { getFinalUrl } from "../../utils/getFinalUrl.js"; 13 + import { GLOBAL_ALLOW } from "../../constants.js"; 13 14 14 15 export const checkPosts = async (post: Post[]) => { 15 16 if (GLOBAL_ALLOW.includes(post[0].did)) { ··· 20 21 return; 21 22 } 22 23 23 - const urlRegex = /https?:\/\/[^\s]+/g; 24 + const urlRegex = /https?:\/\/[^\s]+/gi; 24 25 25 26 // Check for link shorteners 26 27 if (LINK_SHORTENER.test(post[0].text)) { 27 28 try { 28 29 const url = post[0].text.match(urlRegex); 29 30 if (url && LINK_SHORTENER.test(url[0])) { 30 - // logger.info(`[CHECKPOSTS]: Checking shortened URL: ${url[0]}`); 31 + logger.debug( 32 + { process: "CHECKPOSTS", url: url[0], did: post[0].did }, 33 + "Resolving shortened URL", 34 + ); 35 + 31 36 const finalUrl = await getFinalUrl(url[0]); 32 - if (finalUrl) { 33 - const originalUrl = post[0].text; 37 + if (finalUrl && finalUrl !== url[0]) { 34 38 post[0].text = post[0].text.replace(url[0], finalUrl); 35 - /* logger.info( 36 - `[CHECKPOSTS]: Shortened URL resolved: ${originalUrl} -> ${finalUrl}`, 37 - ); */ 39 + logger.debug( 40 + { 41 + process: "CHECKPOSTS", 42 + originalUrl: url[0], 43 + resolvedUrl: finalUrl, 44 + did: post[0].did, 45 + }, 46 + "Shortened URL resolved", 47 + ); 38 48 } 39 49 } 40 50 } catch (error) { 51 + const errorInfo = 52 + error instanceof Error 53 + ? { 54 + name: error.name, 55 + message: error.message, 56 + } 57 + : { error: String(error) }; 58 + 41 59 logger.error( 42 - { process: "CHECKPOSTS", text: post[0].text, error }, 60 + { 61 + process: "CHECKPOSTS", 62 + text: post[0].text, 63 + did: post[0].did, 64 + atURI: post[0].atURI, 65 + ...errorInfo, 66 + }, 43 67 "Failed to resolve shortened URL", 44 68 ); 45 69 // Keep the original URL if resolution fails
+5 -5
src/checkProfiles.ts src/rules/profiles/checkProfiles.ts
··· 1 - import { PROFILE_CHECKS } from "./constants.js"; 2 - import { logger } from "./logger.js"; 1 + import { PROFILE_CHECKS } from "../../rules/profiles/constants.js"; 2 + import { logger } from "../../logger.js"; 3 3 import { 4 4 createAccountReport, 5 5 createAccountLabel, 6 6 createAccountComment, 7 - } from "./moderation.js"; 8 - import { getLanguage } from "./utils.js"; 9 - import { GLOBAL_ALLOW } from "./constants.js"; 7 + } from "../../moderation.js"; 8 + import { getLanguage } from "../../utils/getLanguage.js"; 9 + import { GLOBAL_ALLOW } from "../../constants.js"; 10 10 11 11 export const checkDescription = async ( 12 12 did: string,
-30
src/count.ts
··· 1 - import { isLoggedIn, agent } from "./agent.js"; 2 - import { logger } from "./logger.js"; 3 - import { limit } from "./limits.js"; 4 - import { createAccountLabel } from "./moderation.js"; 5 - 6 - export const countStarterPacks = async (did: string, time: number) => { 7 - await isLoggedIn; 8 - 9 - if (did in ["did:plc:gpunjjgvlyb4racypz3yfiq4"]) { 10 - logger.debug({ process: "COUNTSTARTERPACKS", did, time }, "Account is whitelisted"); 11 - return; 12 - } 13 - 14 - await limit(async () => { 15 - try { 16 - const profile = await agent.app.bsky.actor.getProfile({ actor: did }); 17 - const starterPacks = profile.data.associated?.starterPacks; 18 - 19 - if (starterPacks && starterPacks.valueOf() > 20) { 20 - createAccountLabel( 21 - did, 22 - "follow-farming", 23 - `[COUNTSTARTERPACKS]: ${time}: Account ${did} has ${starterPacks} starter packs.`, 24 - ); 25 - } 26 - } catch (error) { 27 - logger.error({ process: "COUNTSTARTERPACKS", error }, "Error checking associated accounts"); 28 - } 29 - }); 30 - };
+5
src/homoglyphs.ts src/utils/homoglyphs.ts
··· 42 42 a: "a", 43 43 "@": "a", 44 44 "4": "a", 45 + а: "a", // cyrillic a 45 46 46 47 // Confusables for 'e' 47 48 "3": "e", ··· 88 89 ꬴ: "e", 89 90 ꬳ: "e", 90 91 e: "e", 92 + ё: "e", // cyrillic io 91 93 92 94 // Confusables for 'g' 93 95 ǵ: "g", ··· 139 141 ı: "i", 140 142 i: "i", 141 143 "1": "i", 144 + і: "i", // cyrillic i 142 145 ĺ: "l", 143 146 ľ: "l", 144 147 ļ: "l", ··· 252 255 ⱺ: "o", 253 256 o: "o", 254 257 "0": "o", 258 + о: "o", // cyrillic o 255 259 256 260 // Confusables for 'r' 257 261 ŕ: "r", ··· 305 309 ᵵ: "t", 306 310 ƫ: "t", 307 311 ȶ: "t", 312 + т: "t", // cyrillic t 308 313 };
+9 -4
src/main.ts
··· 15 15 import { logger } from "./logger.js"; 16 16 import { startMetricsServer } from "./metrics.js"; 17 17 import { Post, LinkFeature, Handle } from "./types.js"; 18 - import { checkPosts } from "./checkPosts.js"; 19 - import { checkHandle } from "./checkHandles.js"; 20 - import { checkDescription, checkDisplayName } from "./checkProfiles.js"; 18 + import { checkPosts } from "./rules/posts/checkPosts.js"; 19 + import { checkHandle } from "./rules/handles/checkHandles.js"; 20 + import { 21 + checkDescription, 22 + checkDisplayName, 23 + } from "./rules/profiles/checkProfiles.js"; 21 24 import { checkFacetSpam } from "./rules/facets/facets.js"; 22 25 import { checkAccountAge } from "./rules/account/age.js"; 23 26 ··· 139 142 embed.$type === "app.bsky.embed.recordWithMedia") 140 143 ) { 141 144 const record = 142 - embed.$type === "app.bsky.embed.record" ? embed.record : embed.record.record; 145 + embed.$type === "app.bsky.embed.record" 146 + ? embed.record 147 + : embed.record.record; 143 148 if (record && record.uri) { 144 149 const quotedPostURI = record.uri; 145 150 const quotedDid = quotedPostURI.split("/")[2]; // Extract DID from at://did/...
+56
src/rules/account/countStarterPacks.ts
··· 1 + import { isLoggedIn, agent } from "../../agent.js"; 2 + import { logger } from "../../logger.js"; 3 + import { limit } from "../../limits.js"; 4 + import { createAccountLabel } from "../../moderation.js"; 5 + 6 + const ALLOWED_DIDS = ["did:plc:gpunjjgvlyb4racypz3yfiq4"]; 7 + 8 + export const countStarterPacks = async (did: string, time: number) => { 9 + await isLoggedIn; 10 + 11 + if (ALLOWED_DIDS.includes(did)) { 12 + logger.debug( 13 + { process: "COUNTSTARTERPACKS", did, time }, 14 + "Account is whitelisted", 15 + ); 16 + return; 17 + } 18 + 19 + await limit(async () => { 20 + try { 21 + const profile = await agent.app.bsky.actor.getProfile({ actor: did }); 22 + const starterPacks = profile.data.associated?.starterPacks; 23 + 24 + if (starterPacks && starterPacks.valueOf() > 20) { 25 + logger.info( 26 + { 27 + process: "COUNTSTARTERPACKS", 28 + did, 29 + time, 30 + starterPackCount: starterPacks.valueOf(), 31 + }, 32 + "Labeling account with excessive starter packs", 33 + ); 34 + 35 + createAccountLabel( 36 + did, 37 + "follow-farming", 38 + `${time}: Account has ${starterPacks} starter packs`, 39 + ); 40 + } 41 + } catch (error) { 42 + const errorInfo = 43 + error instanceof Error 44 + ? { 45 + name: error.name, 46 + message: error.message, 47 + } 48 + : { error: String(error) }; 49 + 50 + logger.error( 51 + { process: "COUNTSTARTERPACKS", did, time, ...errorInfo }, 52 + "Error checking associated accounts", 53 + ); 54 + } 55 + }); 56 + };
+231
src/rules/account/tests/countStarterPacks.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { countStarterPacks } from "../countStarterPacks.js"; 3 + 4 + // Mock dependencies 5 + vi.mock("../../../agent.js", () => ({ 6 + agent: { 7 + app: { 8 + bsky: { 9 + actor: { 10 + getProfile: vi.fn(), 11 + }, 12 + }, 13 + }, 14 + }, 15 + isLoggedIn: Promise.resolve(true), 16 + })); 17 + 18 + vi.mock("../../../logger.js", () => ({ 19 + logger: { 20 + info: vi.fn(), 21 + debug: vi.fn(), 22 + warn: vi.fn(), 23 + error: vi.fn(), 24 + }, 25 + })); 26 + 27 + vi.mock("../../../moderation.js", () => ({ 28 + createAccountLabel: vi.fn(), 29 + })); 30 + 31 + vi.mock("../../../limits.js", () => ({ 32 + limit: vi.fn((fn) => fn()), 33 + })); 34 + 35 + import { agent } from "../../../agent.js"; 36 + import { logger } from "../../../logger.js"; 37 + import { createAccountLabel } from "../../../moderation.js"; 38 + import { limit } from "../../../limits.js"; 39 + 40 + describe("countStarterPacks", () => { 41 + beforeEach(() => { 42 + vi.clearAllMocks(); 43 + }); 44 + 45 + it("should skip whitelisted DIDs", async () => { 46 + const whitelistedDid = "did:plc:gpunjjgvlyb4racypz3yfiq4"; 47 + const time = Date.now() * 1000; 48 + 49 + await countStarterPacks(whitelistedDid, time); 50 + 51 + expect(logger.debug).toHaveBeenCalledWith( 52 + { process: "COUNTSTARTERPACKS", did: whitelistedDid, time }, 53 + "Account is whitelisted", 54 + ); 55 + expect(agent.app.bsky.actor.getProfile).not.toHaveBeenCalled(); 56 + expect(createAccountLabel).not.toHaveBeenCalled(); 57 + }); 58 + 59 + it("should label accounts with more than 20 starter packs", async () => { 60 + const did = "did:plc:test123"; 61 + const time = Date.now() * 1000; 62 + const starterPackCount = 25; 63 + 64 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 65 + data: { 66 + associated: { 67 + starterPacks: starterPackCount, 68 + }, 69 + }, 70 + } as any); 71 + 72 + await countStarterPacks(did, time); 73 + 74 + expect(limit).toHaveBeenCalled(); 75 + expect(agent.app.bsky.actor.getProfile).toHaveBeenCalledWith({ 76 + actor: did, 77 + }); 78 + expect(logger.info).toHaveBeenCalledWith( 79 + { 80 + process: "COUNTSTARTERPACKS", 81 + did, 82 + time, 83 + starterPackCount, 84 + }, 85 + "Labeling account with excessive starter packs", 86 + ); 87 + expect(createAccountLabel).toHaveBeenCalledWith( 88 + did, 89 + "follow-farming", 90 + `${time}: Account has ${starterPackCount} starter packs`, 91 + ); 92 + }); 93 + 94 + it("should not label accounts with exactly 20 starter packs", async () => { 95 + const did = "did:plc:test456"; 96 + const time = Date.now() * 1000; 97 + 98 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 99 + data: { 100 + associated: { 101 + starterPacks: 20, 102 + }, 103 + }, 104 + } as any); 105 + 106 + await countStarterPacks(did, time); 107 + 108 + expect(agent.app.bsky.actor.getProfile).toHaveBeenCalledWith({ 109 + actor: did, 110 + }); 111 + expect(createAccountLabel).not.toHaveBeenCalled(); 112 + }); 113 + 114 + it("should not label accounts with fewer than 20 starter packs", async () => { 115 + const did = "did:plc:test789"; 116 + const time = Date.now() * 1000; 117 + 118 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 119 + data: { 120 + associated: { 121 + starterPacks: 15, 122 + }, 123 + }, 124 + } as any); 125 + 126 + await countStarterPacks(did, time); 127 + 128 + expect(agent.app.bsky.actor.getProfile).toHaveBeenCalledWith({ 129 + actor: did, 130 + }); 131 + expect(createAccountLabel).not.toHaveBeenCalled(); 132 + }); 133 + 134 + it("should not label accounts with no associated starter packs", async () => { 135 + const did = "did:plc:test000"; 136 + const time = Date.now() * 1000; 137 + 138 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 139 + data: { 140 + associated: undefined, 141 + }, 142 + } as any); 143 + 144 + await countStarterPacks(did, time); 145 + 146 + expect(agent.app.bsky.actor.getProfile).toHaveBeenCalledWith({ 147 + actor: did, 148 + }); 149 + expect(createAccountLabel).not.toHaveBeenCalled(); 150 + }); 151 + 152 + it("should not label accounts with no starter packs field", async () => { 153 + const did = "did:plc:test111"; 154 + const time = Date.now() * 1000; 155 + 156 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 157 + data: { 158 + associated: { 159 + starterPacks: undefined, 160 + }, 161 + }, 162 + } as any); 163 + 164 + await countStarterPacks(did, time); 165 + 166 + expect(agent.app.bsky.actor.getProfile).toHaveBeenCalledWith({ 167 + actor: did, 168 + }); 169 + expect(createAccountLabel).not.toHaveBeenCalled(); 170 + }); 171 + 172 + it("should handle errors when fetching profile", async () => { 173 + const did = "did:plc:testerror"; 174 + const time = Date.now() * 1000; 175 + const error = new Error("Profile not found"); 176 + 177 + vi.mocked(agent.app.bsky.actor.getProfile).mockRejectedValue(error); 178 + 179 + await countStarterPacks(did, time); 180 + 181 + expect(logger.error).toHaveBeenCalledWith( 182 + { 183 + process: "COUNTSTARTERPACKS", 184 + did, 185 + time, 186 + name: error.name, 187 + message: error.message, 188 + }, 189 + "Error checking associated accounts", 190 + ); 191 + expect(createAccountLabel).not.toHaveBeenCalled(); 192 + }); 193 + 194 + it("should handle non-Error exceptions", async () => { 195 + const did = "did:plc:teststring"; 196 + const time = Date.now() * 1000; 197 + const error = "String error message"; 198 + 199 + vi.mocked(agent.app.bsky.actor.getProfile).mockRejectedValue(error); 200 + 201 + await countStarterPacks(did, time); 202 + 203 + expect(logger.error).toHaveBeenCalledWith( 204 + { 205 + process: "COUNTSTARTERPACKS", 206 + did, 207 + time, 208 + error: String(error), 209 + }, 210 + "Error checking associated accounts", 211 + ); 212 + expect(createAccountLabel).not.toHaveBeenCalled(); 213 + }); 214 + 215 + it("should use limit function for rate limiting", async () => { 216 + const did = "did:plc:testlimit"; 217 + const time = Date.now() * 1000; 218 + 219 + vi.mocked(agent.app.bsky.actor.getProfile).mockResolvedValue({ 220 + data: { 221 + associated: { 222 + starterPacks: 10, 223 + }, 224 + }, 225 + } as any); 226 + 227 + await countStarterPacks(did, time); 228 + 229 + expect(limit).toHaveBeenCalledWith(expect.any(Function)); 230 + }); 231 + });
+332
src/rules/handles/checkHandles.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { checkHandle } from "./checkHandles.js"; 3 + import { 4 + createAccountReport, 5 + createAccountComment, 6 + createAccountLabel, 7 + } from "../../moderation.js"; 8 + 9 + // Mock dependencies 10 + vi.mock("../../moderation.js", () => ({ 11 + createAccountReport: vi.fn(), 12 + createAccountComment: vi.fn(), 13 + createAccountLabel: vi.fn(), 14 + })); 15 + 16 + vi.mock("../../logger.js", () => ({ 17 + logger: { 18 + debug: vi.fn(), 19 + info: vi.fn(), 20 + warn: vi.fn(), 21 + }, 22 + })); 23 + 24 + vi.mock("../../constants.js", () => ({ 25 + GLOBAL_ALLOW: ["did:plc:globalallow"], 26 + })); 27 + 28 + // Mock HANDLE_CHECKS with various test scenarios 29 + vi.mock("./constants.js", () => ({ 30 + HANDLE_CHECKS: [ 31 + { 32 + label: "spam", 33 + comment: "Spam detected", 34 + reportAcct: true, 35 + commentAcct: false, 36 + toLabel: false, 37 + check: /spam/i, 38 + }, 39 + { 40 + label: "scam", 41 + comment: "Scam detected", 42 + reportAcct: false, 43 + commentAcct: true, 44 + toLabel: false, 45 + check: /scam/i, 46 + whitelist: /legit-scam/i, 47 + }, 48 + { 49 + label: "bot", 50 + comment: "Bot detected", 51 + reportAcct: false, 52 + commentAcct: false, 53 + toLabel: true, 54 + check: /bot-\d+/i, 55 + ignoredDIDs: ["did:plc:ignoredbot"], 56 + }, 57 + { 58 + label: "multi-action", 59 + comment: "Multi-action triggered", 60 + reportAcct: true, 61 + commentAcct: true, 62 + toLabel: true, 63 + check: /dangerous/i, 64 + }, 65 + { 66 + label: "whitelist-test", 67 + comment: "Whitelisted pattern", 68 + reportAcct: true, 69 + commentAcct: false, 70 + toLabel: false, 71 + check: /test/i, 72 + whitelist: /good-test/i, 73 + ignoredDIDs: ["did:plc:testuser"], 74 + }, 75 + ], 76 + })); 77 + 78 + describe("checkHandle", () => { 79 + beforeEach(() => { 80 + vi.clearAllMocks(); 81 + }); 82 + 83 + describe("global allow list", () => { 84 + it("should skip checks for globally allowed DIDs", async () => { 85 + await checkHandle("did:plc:globalallow", "spam-account", Date.now()); 86 + 87 + expect(createAccountReport).not.toHaveBeenCalled(); 88 + expect(createAccountComment).not.toHaveBeenCalled(); 89 + expect(createAccountLabel).not.toHaveBeenCalled(); 90 + }); 91 + 92 + it("should process non-globally-allowed DIDs", async () => { 93 + await checkHandle("did:plc:normal", "spam-account", Date.now()); 94 + 95 + expect(createAccountReport).toHaveBeenCalled(); 96 + }); 97 + }); 98 + 99 + describe("pattern matching", () => { 100 + it("should trigger on matching pattern", async () => { 101 + const time = Date.now(); 102 + await checkHandle("did:plc:user1", "spam-account", time); 103 + 104 + expect(createAccountReport).toHaveBeenCalledWith( 105 + "did:plc:user1", 106 + `${time}: Spam detected - spam-account`, 107 + ); 108 + }); 109 + 110 + it("should not trigger on non-matching pattern", async () => { 111 + await checkHandle("did:plc:user1", "normal-account", Date.now()); 112 + 113 + expect(createAccountReport).not.toHaveBeenCalled(); 114 + expect(createAccountComment).not.toHaveBeenCalled(); 115 + expect(createAccountLabel).not.toHaveBeenCalled(); 116 + }); 117 + 118 + it("should be case insensitive", async () => { 119 + const time = Date.now(); 120 + await checkHandle("did:plc:user1", "SPAM-ACCOUNT", time); 121 + 122 + expect(createAccountReport).toHaveBeenCalledWith( 123 + "did:plc:user1", 124 + `${time}: Spam detected - SPAM-ACCOUNT`, 125 + ); 126 + }); 127 + }); 128 + 129 + describe("whitelist handling", () => { 130 + it("should not trigger on whitelisted pattern", async () => { 131 + await checkHandle("did:plc:user1", "legit-scam-detector", Date.now()); 132 + 133 + expect(createAccountComment).not.toHaveBeenCalled(); 134 + }); 135 + 136 + it("should trigger on non-whitelisted match", async () => { 137 + const time = Date.now(); 138 + await checkHandle("did:plc:user1", "scam-account", time); 139 + 140 + expect(createAccountComment).toHaveBeenCalledWith( 141 + "did:plc:user1", 142 + `${time}: Scam detected - scam-account`, 143 + ); 144 + }); 145 + }); 146 + 147 + describe("ignored DIDs", () => { 148 + it("should skip checks for ignored DIDs in specific rules", async () => { 149 + await checkHandle("did:plc:ignoredbot", "bot-123", Date.now()); 150 + 151 + expect(createAccountLabel).not.toHaveBeenCalled(); 152 + }); 153 + 154 + it("should process non-ignored DIDs for specific rules", async () => { 155 + const time = Date.now(); 156 + await checkHandle("did:plc:normaluser", "bot-456", time); 157 + 158 + expect(createAccountLabel).toHaveBeenCalledWith( 159 + "did:plc:normaluser", 160 + "bot", 161 + `${time}: Bot detected - bot-456`, 162 + ); 163 + }); 164 + }); 165 + 166 + describe("action types", () => { 167 + it("should create report when reportAcct is true", async () => { 168 + const time = Date.now(); 169 + await checkHandle("did:plc:user1", "spam-user", time); 170 + 171 + expect(createAccountReport).toHaveBeenCalledWith( 172 + "did:plc:user1", 173 + `${time}: Spam detected - spam-user`, 174 + ); 175 + }); 176 + 177 + it("should create comment when commentAcct is true", async () => { 178 + const time = Date.now(); 179 + await checkHandle("did:plc:user1", "scam-user", time); 180 + 181 + expect(createAccountComment).toHaveBeenCalledWith( 182 + "did:plc:user1", 183 + `${time}: Scam detected - scam-user`, 184 + ); 185 + }); 186 + 187 + it("should create label when toLabel is true", async () => { 188 + const time = Date.now(); 189 + await checkHandle("did:plc:user1", "bot-789", time); 190 + 191 + expect(createAccountLabel).toHaveBeenCalledWith( 192 + "did:plc:user1", 193 + "bot", 194 + `${time}: Bot detected - bot-789`, 195 + ); 196 + }); 197 + 198 + it("should perform multiple actions when configured", async () => { 199 + const time = Date.now(); 200 + await checkHandle("did:plc:user1", "dangerous-account", time); 201 + 202 + expect(createAccountReport).toHaveBeenCalledWith( 203 + "did:plc:user1", 204 + `${time}: Multi-action triggered - dangerous-account`, 205 + ); 206 + expect(createAccountComment).toHaveBeenCalledWith( 207 + "did:plc:user1", 208 + `${time}: Multi-action triggered - dangerous-account`, 209 + ); 210 + expect(createAccountLabel).toHaveBeenCalledWith( 211 + "did:plc:user1", 212 + "multi-action", 213 + `${time}: Multi-action triggered - dangerous-account`, 214 + ); 215 + }); 216 + }); 217 + 218 + describe("multiple rule matching", () => { 219 + it("should process all matching rules", async () => { 220 + vi.resetModules(); 221 + // Re-import with a mock that has overlapping patterns 222 + vi.doMock("./constants.js", () => ({ 223 + HANDLE_CHECKS: [ 224 + { 225 + label: "pattern1", 226 + comment: "Pattern 1", 227 + reportAcct: true, 228 + commentAcct: false, 229 + toLabel: false, 230 + check: /test/i, 231 + }, 232 + { 233 + label: "pattern2", 234 + comment: "Pattern 2", 235 + reportAcct: false, 236 + commentAcct: true, 237 + toLabel: false, 238 + check: /test/i, 239 + }, 240 + ], 241 + })); 242 + 243 + const { checkHandle: checkHandleReimport } = await import( 244 + "./checkHandles.js" 245 + ); 246 + const time = Date.now(); 247 + await checkHandleReimport("did:plc:user1", "test-account", time); 248 + 249 + expect(createAccountReport).toHaveBeenCalledTimes(1); 250 + expect(createAccountComment).toHaveBeenCalledTimes(1); 251 + }); 252 + }); 253 + 254 + describe("edge cases", () => { 255 + it("should handle empty handle strings", async () => { 256 + await checkHandle("did:plc:user1", "", Date.now()); 257 + 258 + expect(createAccountReport).not.toHaveBeenCalled(); 259 + expect(createAccountComment).not.toHaveBeenCalled(); 260 + expect(createAccountLabel).not.toHaveBeenCalled(); 261 + }); 262 + 263 + it("should handle special characters in handles", async () => { 264 + await checkHandle( 265 + "did:plc:user1", 266 + "spam-!@#$%^&*()", 267 + Date.now(), 268 + ); 269 + 270 + expect(createAccountReport).toHaveBeenCalled(); 271 + }); 272 + 273 + it("should handle very long handles", async () => { 274 + const longHandle = "spam-" + "a".repeat(1000); 275 + const time = Date.now(); 276 + await checkHandle("did:plc:user1", longHandle, time); 277 + 278 + expect(createAccountReport).toHaveBeenCalledWith( 279 + "did:plc:user1", 280 + `${time}: Spam detected - ${longHandle}`, 281 + ); 282 + }); 283 + 284 + it("should handle unicode characters in handles", async () => { 285 + await checkHandle("did:plc:user1", "spam-账户-🤖", Date.now()); 286 + 287 + expect(createAccountReport).toHaveBeenCalled(); 288 + }); 289 + }); 290 + 291 + describe("timestamp handling", () => { 292 + it("should include timestamp in action comments", async () => { 293 + const time = 1234567890; 294 + await checkHandle("did:plc:user1", "spam-account", time); 295 + 296 + expect(createAccountReport).toHaveBeenCalledWith( 297 + "did:plc:user1", 298 + "1234567890: Spam detected - spam-account", 299 + ); 300 + }); 301 + 302 + it("should handle different timestamp formats", async () => { 303 + const time = Date.now(); 304 + await checkHandle("did:plc:user1", "spam-account", time); 305 + 306 + expect(createAccountReport).toHaveBeenCalledWith( 307 + "did:plc:user1", 308 + expect.stringContaining(time.toString()), 309 + ); 310 + }); 311 + }); 312 + 313 + describe("whitelist and ignoredDIDs combination", () => { 314 + it("should respect both whitelist and ignoredDIDs", async () => { 315 + await checkHandle("did:plc:testuser", "good-test-account", Date.now()); 316 + 317 + expect(createAccountReport).not.toHaveBeenCalled(); 318 + }); 319 + 320 + it("should skip on ignoredDID even if not whitelisted", async () => { 321 + await checkHandle("did:plc:testuser", "bad-test-account", Date.now()); 322 + 323 + expect(createAccountReport).not.toHaveBeenCalled(); 324 + }); 325 + 326 + it("should skip on whitelist even if not in ignoredDIDs", async () => { 327 + await checkHandle("did:plc:otheruser", "good-test-account", Date.now()); 328 + 329 + expect(createAccountReport).not.toHaveBeenCalled(); 330 + }); 331 + }); 332 + });
+92
src/rules/handles/constants.example.ts
··· 1 + import { Checks } from "../../types.js"; 2 + 3 + /** 4 + * Example handle check configurations 5 + * 6 + * This file demonstrates how to configure handle-based moderation rules. 7 + * Copy this file to constants.ts and customize for your labeler's needs. 8 + * 9 + * Each check can match against handles, display names, and/or descriptions 10 + * based on the flags you set (description: true, displayName: true). 11 + */ 12 + 13 + export const HANDLE_CHECKS: Checks[] = [ 14 + // Example 1: Simple pattern matching with whitelist 15 + { 16 + label: "spam-indicator", 17 + comment: "Handle matches common spam patterns", 18 + reportAcct: false, 19 + commentAcct: false, 20 + toLabel: true, 21 + check: new RegExp( 22 + "follow.*?back|gain.*?followers|crypto.*?giveaway|free.*?money", 23 + "i", 24 + ), 25 + whitelist: new RegExp("legitimate.*?business", "i"), 26 + }, 27 + 28 + // Example 2: Check specific domain patterns 29 + { 30 + label: "suspicious-domain", 31 + comment: "Handle uses suspicious domain pattern", 32 + reportAcct: false, 33 + commentAcct: false, 34 + toLabel: true, 35 + check: new RegExp("(?:suspicious-site\\.example)", "i"), 36 + }, 37 + 38 + // Example 3: Check with display name and description matching 39 + { 40 + label: "potential-impersonator", 41 + comment: "Account may be impersonating verified entities", 42 + description: true, 43 + displayName: true, 44 + reportAcct: false, 45 + commentAcct: false, 46 + toLabel: true, 47 + check: new RegExp( 48 + "official.*?support|customer.*?service.*?rep|verified.*?account", 49 + "i", 50 + ), 51 + // Exclude accounts that are actually legitimate 52 + ignoredDIDs: [ 53 + "did:plc:example123", // Real customer support account 54 + "did:plc:example456", // Verified business account 55 + ], 56 + }, 57 + 58 + // Example 4: Pattern with specific character variations 59 + { 60 + label: "suspicious-pattern", 61 + comment: "Handle contains suspicious character patterns", 62 + reportAcct: false, 63 + commentAcct: false, 64 + toLabel: true, 65 + check: new RegExp( 66 + "[a-z]{2,}[0-9]{6,}|random.*?numbers.*?[0-9]{4,}", 67 + "i", 68 + ), 69 + whitelist: new RegExp("year[0-9]{4}", "i"), 70 + ignoredDIDs: [ 71 + "did:plc:example789", // Legitimate account with number pattern 72 + ], 73 + }, 74 + 75 + // Example 5: Brand protection 76 + { 77 + label: "brand-impersonation", 78 + comment: "Potential brand impersonation detected", 79 + reportAcct: false, 80 + commentAcct: false, 81 + toLabel: true, 82 + check: new RegExp("example-?brand|cool-?company|awesome-?corp", "i"), 83 + whitelist: new RegExp( 84 + "anti-example-brand|not-cool-company|parody.*awesome-corp", 85 + "i", 86 + ), 87 + ignoredDIDs: [ 88 + "did:plc:exampleabc", // Official brand account 89 + "did:plc:exampledef", // Authorized partner 90 + ], 91 + }, 92 + ];
src/rules/posts/constants.example.ts

This is a binary file and will not be displayed.

+446
src/rules/posts/tests/checkPosts.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { checkPosts } from "../checkPosts.js"; 3 + import { Post } from "../../../types.js"; 4 + 5 + // Mock dependencies 6 + vi.mock("../constants.js", () => ({ 7 + LINK_SHORTENER: /tinyurl\.com|bit\.ly/i, 8 + POST_CHECKS: [ 9 + { 10 + label: "test-label", 11 + comment: "Test comment", 12 + check: /spam|scam/i, 13 + toLabel: true, 14 + reportPost: false, 15 + reportAcct: false, 16 + commentAcct: false, 17 + }, 18 + { 19 + label: "language-specific", 20 + comment: "English only test", 21 + language: ["eng"], 22 + check: /hello/i, 23 + toLabel: true, 24 + reportPost: false, 25 + reportAcct: false, 26 + commentAcct: false, 27 + }, 28 + { 29 + label: "whitelisted-test", 30 + comment: "Has whitelist", 31 + check: /bad/i, 32 + whitelist: /good bad/i, 33 + toLabel: true, 34 + reportPost: false, 35 + reportAcct: false, 36 + commentAcct: false, 37 + }, 38 + { 39 + label: "ignored-did", 40 + comment: "Ignored DID test", 41 + check: /test/i, 42 + ignoredDIDs: ["did:plc:ignored123"], 43 + toLabel: true, 44 + reportPost: false, 45 + reportAcct: false, 46 + commentAcct: false, 47 + }, 48 + { 49 + label: "all-actions", 50 + comment: "All actions enabled", 51 + check: /report/i, 52 + toLabel: true, 53 + reportPost: true, 54 + reportAcct: true, 55 + commentAcct: true, 56 + }, 57 + ], 58 + })); 59 + 60 + vi.mock("../../../logger.js", () => ({ 61 + logger: { 62 + info: vi.fn(), 63 + debug: vi.fn(), 64 + warn: vi.fn(), 65 + error: vi.fn(), 66 + }, 67 + })); 68 + 69 + vi.mock("../../account/countStarterPacks.js", () => ({ 70 + countStarterPacks: vi.fn(), 71 + })); 72 + 73 + vi.mock("../../../moderation.js", () => ({ 74 + createPostLabel: vi.fn(), 75 + createAccountReport: vi.fn(), 76 + createAccountComment: vi.fn(), 77 + createPostReport: vi.fn(), 78 + })); 79 + 80 + vi.mock("../../../utils/getLanguage.js", () => ({ 81 + getLanguage: vi.fn().mockResolvedValue("eng"), 82 + })); 83 + 84 + vi.mock("../../../utils/getFinalUrl.js", () => ({ 85 + getFinalUrl: vi.fn(), 86 + })); 87 + 88 + vi.mock("../../../constants.js", () => ({ 89 + GLOBAL_ALLOW: ["did:plc:globalallow"], 90 + })); 91 + 92 + import { logger } from "../../../logger.js"; 93 + import { countStarterPacks } from "../../account/countStarterPacks.js"; 94 + import { 95 + createPostLabel, 96 + createAccountReport, 97 + createAccountComment, 98 + createPostReport, 99 + } from "../../../moderation.js"; 100 + import { getLanguage } from "../../../utils/getLanguage.js"; 101 + import { getFinalUrl } from "../../../utils/getFinalUrl.js"; 102 + 103 + describe("checkPosts", () => { 104 + beforeEach(() => { 105 + vi.clearAllMocks(); 106 + }); 107 + 108 + const createMockPost = (overrides?: Partial<Post>): Post[] => [ 109 + { 110 + did: "did:plc:test123", 111 + time: Date.now() * 1000, 112 + rkey: "test-rkey", 113 + atURI: "at://did:plc:test123/app.bsky.feed.post/test-rkey", 114 + text: "This is a test post", 115 + cid: "test-cid", 116 + ...overrides, 117 + }, 118 + ]; 119 + 120 + describe("Global allow list", () => { 121 + it("should skip posts from globally allowed DIDs", async () => { 122 + const post = createMockPost({ did: "did:plc:globalallow" }); 123 + 124 + await checkPosts(post); 125 + 126 + expect(logger.warn).toHaveBeenCalledWith( 127 + { 128 + process: "CHECKPOSTS", 129 + did: "did:plc:globalallow", 130 + atURI: post[0].atURI, 131 + }, 132 + "Global AllowListed DID", 133 + ); 134 + expect(createPostLabel).not.toHaveBeenCalled(); 135 + }); 136 + 137 + it("should process posts from non-globally-allowed DIDs", async () => { 138 + const post = createMockPost({ text: "spam post" }); 139 + 140 + await checkPosts(post); 141 + 142 + expect(logger.warn).not.toHaveBeenCalledWith( 143 + expect.objectContaining({ did: post[0].did }), 144 + "Global AllowListed DID", 145 + ); 146 + }); 147 + }); 148 + 149 + describe("URL shortener resolution", () => { 150 + it("should resolve shortened URLs", async () => { 151 + const post = createMockPost({ text: "Check this out https://tinyurl.com/test123" }); 152 + vi.mocked(getFinalUrl).mockResolvedValue("https://example.com/full-url"); 153 + 154 + await checkPosts(post); 155 + 156 + expect(logger.debug).toHaveBeenCalledWith( 157 + { 158 + process: "CHECKPOSTS", 159 + url: "https://tinyurl.com/test123", 160 + did: post[0].did, 161 + }, 162 + "Resolving shortened URL", 163 + ); 164 + expect(getFinalUrl).toHaveBeenCalledWith("https://tinyurl.com/test123"); 165 + expect(logger.debug).toHaveBeenCalledWith( 166 + { 167 + process: "CHECKPOSTS", 168 + originalUrl: "https://tinyurl.com/test123", 169 + resolvedUrl: "https://example.com/full-url", 170 + did: post[0].did, 171 + }, 172 + "Shortened URL resolved", 173 + ); 174 + expect(post[0].text).toBe("Check this out https://example.com/full-url"); 175 + }); 176 + 177 + it("should not replace URL if resolution returns same URL", async () => { 178 + const post = createMockPost({ text: "Check https://tinyurl.com/test" }); 179 + vi.mocked(getFinalUrl).mockResolvedValue("https://tinyurl.com/test"); 180 + 181 + await checkPosts(post); 182 + 183 + expect(getFinalUrl).toHaveBeenCalled(); 184 + expect(post[0].text).toBe("Check https://tinyurl.com/test"); 185 + expect(logger.debug).not.toHaveBeenCalledWith( 186 + expect.objectContaining({ originalUrl: expect.anything() }), 187 + "Shortened URL resolved", 188 + ); 189 + }); 190 + 191 + it("should handle URL resolution errors gracefully", async () => { 192 + const post = createMockPost({ text: "https://tinyurl.com/broken" }); 193 + const error = new Error("Network timeout"); 194 + vi.mocked(getFinalUrl).mockRejectedValue(error); 195 + 196 + await checkPosts(post); 197 + 198 + expect(logger.error).toHaveBeenCalledWith( 199 + { 200 + process: "CHECKPOSTS", 201 + text: post[0].text, 202 + did: post[0].did, 203 + atURI: post[0].atURI, 204 + name: error.name, 205 + message: error.message, 206 + }, 207 + "Failed to resolve shortened URL", 208 + ); 209 + expect(post[0].text).toBe("https://tinyurl.com/broken"); 210 + }); 211 + 212 + it("should handle non-Error exceptions during URL resolution", async () => { 213 + const post = createMockPost({ text: "https://bit.ly/test" }); 214 + const error = "String error"; 215 + vi.mocked(getFinalUrl).mockRejectedValue(error); 216 + 217 + await checkPosts(post); 218 + 219 + expect(logger.error).toHaveBeenCalledWith( 220 + { 221 + process: "CHECKPOSTS", 222 + text: post[0].text, 223 + did: post[0].did, 224 + atURI: post[0].atURI, 225 + error: String(error), 226 + }, 227 + "Failed to resolve shortened URL", 228 + ); 229 + }); 230 + 231 + it("should not attempt to resolve non-shortened URLs", async () => { 232 + const post = createMockPost({ text: "Normal https://example.com URL" }); 233 + 234 + await checkPosts(post); 235 + 236 + expect(getFinalUrl).not.toHaveBeenCalled(); 237 + }); 238 + }); 239 + 240 + describe("Pattern matching and labeling", () => { 241 + it("should label posts matching check patterns", async () => { 242 + const post = createMockPost({ text: "This is spam content" }); 243 + 244 + await checkPosts(post); 245 + 246 + expect(logger.info).toHaveBeenCalledWith( 247 + { 248 + process: "CHECKPOSTS", 249 + label: "test-label", 250 + did: post[0].did, 251 + atURI: post[0].atURI, 252 + }, 253 + "Labeling post", 254 + ); 255 + expect(createPostLabel).toHaveBeenCalledWith( 256 + post[0].atURI, 257 + post[0].cid, 258 + "test-label", 259 + expect.stringContaining("Test comment"), 260 + undefined, 261 + ); 262 + }); 263 + 264 + it("should not label posts that don't match patterns", async () => { 265 + const post = createMockPost({ text: "Totally normal content" }); 266 + 267 + await checkPosts(post); 268 + 269 + expect(createPostLabel).not.toHaveBeenCalled(); 270 + }); 271 + 272 + it("should call countStarterPacks when pattern matches", async () => { 273 + const post = createMockPost({ text: "spam message" }); 274 + 275 + await checkPosts(post); 276 + 277 + expect(countStarterPacks).toHaveBeenCalledWith(post[0].did, post[0].time); 278 + }); 279 + }); 280 + 281 + describe("Language filtering", () => { 282 + it("should only check language-specific patterns for matching languages", async () => { 283 + const post = createMockPost({ text: "hello world" }); 284 + vi.mocked(getLanguage).mockResolvedValue("eng"); 285 + 286 + await checkPosts(post); 287 + 288 + expect(createPostLabel).toHaveBeenCalledWith( 289 + post[0].atURI, 290 + post[0].cid, 291 + "language-specific", 292 + expect.any(String), 293 + undefined, 294 + ); 295 + }); 296 + 297 + it("should skip language-specific patterns for non-matching languages", async () => { 298 + const post = createMockPost({ text: "hello world" }); 299 + vi.mocked(getLanguage).mockResolvedValue("spa"); 300 + 301 + await checkPosts(post); 302 + 303 + expect(createPostLabel).not.toHaveBeenCalledWith( 304 + expect.any(String), 305 + expect.any(String), 306 + "language-specific", 307 + expect.any(String), 308 + undefined, 309 + ); 310 + }); 311 + }); 312 + 313 + describe("Whitelist handling", () => { 314 + it("should skip patterns when whitelist matches", async () => { 315 + const post = createMockPost({ text: "this is good bad content" }); 316 + 317 + await checkPosts(post); 318 + 319 + expect(logger.debug).toHaveBeenCalledWith( 320 + { 321 + process: "CHECKPOSTS", 322 + did: post[0].did, 323 + atURI: post[0].atURI, 324 + }, 325 + "Whitelisted phrase found", 326 + ); 327 + expect(createPostLabel).not.toHaveBeenCalledWith( 328 + expect.any(String), 329 + expect.any(String), 330 + "whitelisted-test", 331 + expect.any(String), 332 + undefined, 333 + ); 334 + }); 335 + 336 + it("should label when pattern matches but whitelist doesn't", async () => { 337 + const post = createMockPost({ text: "just bad content" }); 338 + 339 + await checkPosts(post); 340 + 341 + expect(createPostLabel).toHaveBeenCalledWith( 342 + post[0].atURI, 343 + post[0].cid, 344 + "whitelisted-test", 345 + expect.any(String), 346 + undefined, 347 + ); 348 + }); 349 + }); 350 + 351 + describe("Ignored DIDs", () => { 352 + it("should skip checks for ignored DIDs", async () => { 353 + const post = createMockPost({ 354 + did: "did:plc:ignored123", 355 + text: "test content", 356 + }); 357 + 358 + await checkPosts(post); 359 + 360 + expect(logger.debug).toHaveBeenCalledWith( 361 + { 362 + process: "CHECKPOSTS", 363 + did: "did:plc:ignored123", 364 + atURI: post[0].atURI, 365 + }, 366 + "Whitelisted DID", 367 + ); 368 + expect(createPostLabel).not.toHaveBeenCalledWith( 369 + expect.any(String), 370 + expect.any(String), 371 + "ignored-did", 372 + expect.any(String), 373 + undefined, 374 + ); 375 + }); 376 + 377 + it("should check non-ignored DIDs", async () => { 378 + const post = createMockPost({ 379 + did: "did:plc:notignored", 380 + text: "test content", 381 + }); 382 + 383 + await checkPosts(post); 384 + 385 + expect(createPostLabel).toHaveBeenCalledWith( 386 + post[0].atURI, 387 + post[0].cid, 388 + "ignored-did", 389 + expect.any(String), 390 + undefined, 391 + ); 392 + }); 393 + }); 394 + 395 + describe("Moderation actions", () => { 396 + it("should execute all moderation actions when enabled", async () => { 397 + const post = createMockPost({ text: "report this content" }); 398 + 399 + await checkPosts(post); 400 + 401 + expect(createPostLabel).toHaveBeenCalledWith( 402 + post[0].atURI, 403 + post[0].cid, 404 + "all-actions", 405 + expect.any(String), 406 + undefined, 407 + ); 408 + expect(createPostReport).toHaveBeenCalledWith( 409 + post[0].atURI, 410 + post[0].cid, 411 + expect.any(String), 412 + ); 413 + expect(createAccountReport).toHaveBeenCalledWith( 414 + post[0].did, 415 + expect.any(String), 416 + ); 417 + expect(createAccountComment).toHaveBeenCalledWith( 418 + post[0].did, 419 + expect.any(String), 420 + ); 421 + }); 422 + 423 + it("should log all moderation actions", async () => { 424 + const post = createMockPost({ text: "report this" }); 425 + 426 + await checkPosts(post); 427 + 428 + expect(logger.info).toHaveBeenCalledWith( 429 + expect.objectContaining({ label: "all-actions" }), 430 + "Labeling post", 431 + ); 432 + expect(logger.info).toHaveBeenCalledWith( 433 + expect.objectContaining({ label: "all-actions" }), 434 + "Reporting post", 435 + ); 436 + expect(logger.info).toHaveBeenCalledWith( 437 + expect.objectContaining({ label: "all-actions" }), 438 + "Reporting account", 439 + ); 440 + expect(logger.info).toHaveBeenCalledWith( 441 + expect.objectContaining({ label: "all-actions" }), 442 + "Commenting on account", 443 + ); 444 + }); 445 + }); 446 + });
src/rules/profiles/constants.example.ts

This is a binary file and will not be displayed.

+632
src/rules/profiles/tests/checkProfiles.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { checkDescription, checkDisplayName } from "../checkProfiles.js"; 3 + 4 + // Mock dependencies 5 + vi.mock("../constants.js", () => ({ 6 + PROFILE_CHECKS: [ 7 + { 8 + label: "test-description", 9 + comment: "Test description check", 10 + description: true, 11 + displayName: false, 12 + check: /spam|scam/i, 13 + toLabel: true, 14 + reportAcct: false, 15 + commentAcct: false, 16 + }, 17 + { 18 + label: "test-displayname", 19 + comment: "Test display name check", 20 + description: false, 21 + displayName: true, 22 + check: /fake|bot/i, 23 + toLabel: true, 24 + reportAcct: false, 25 + commentAcct: false, 26 + }, 27 + { 28 + label: "language-specific", 29 + comment: "English only test", 30 + language: ["eng"], 31 + description: true, 32 + displayName: false, 33 + check: /hello/i, 34 + toLabel: true, 35 + reportAcct: false, 36 + commentAcct: false, 37 + }, 38 + { 39 + label: "whitelisted-test", 40 + comment: "Has whitelist", 41 + description: true, 42 + displayName: false, 43 + check: /bad/i, 44 + whitelist: /good bad/i, 45 + toLabel: true, 46 + reportAcct: false, 47 + commentAcct: false, 48 + }, 49 + { 50 + label: "ignored-did", 51 + comment: "Ignored DID test", 52 + description: true, 53 + displayName: false, 54 + check: /test/i, 55 + ignoredDIDs: ["did:plc:ignored123"], 56 + toLabel: true, 57 + reportAcct: false, 58 + commentAcct: false, 59 + }, 60 + { 61 + label: "all-actions", 62 + comment: "All actions enabled", 63 + description: true, 64 + displayName: false, 65 + check: /report/i, 66 + toLabel: true, 67 + reportAcct: true, 68 + commentAcct: true, 69 + }, 70 + { 71 + label: "both-fields", 72 + comment: "Check both description and displayName", 73 + description: true, 74 + displayName: true, 75 + check: /suspicious/i, 76 + toLabel: true, 77 + reportAcct: false, 78 + commentAcct: false, 79 + }, 80 + ], 81 + })); 82 + 83 + vi.mock("../../../logger.js", () => ({ 84 + logger: { 85 + info: vi.fn(), 86 + debug: vi.fn(), 87 + warn: vi.fn(), 88 + error: vi.fn(), 89 + }, 90 + })); 91 + 92 + vi.mock("../../../moderation.js", () => ({ 93 + createAccountLabel: vi.fn(), 94 + createAccountReport: vi.fn(), 95 + createAccountComment: vi.fn(), 96 + })); 97 + 98 + vi.mock("../../../utils/getLanguage.js", () => ({ 99 + getLanguage: vi.fn().mockResolvedValue("eng"), 100 + })); 101 + 102 + vi.mock("../../../constants.js", () => ({ 103 + GLOBAL_ALLOW: ["did:plc:globalallow"], 104 + })); 105 + 106 + import { logger } from "../../../logger.js"; 107 + import { 108 + createAccountLabel, 109 + createAccountReport, 110 + createAccountComment, 111 + } from "../../../moderation.js"; 112 + import { getLanguage } from "../../../utils/getLanguage.js"; 113 + 114 + describe("checkProfiles", () => { 115 + beforeEach(() => { 116 + vi.clearAllMocks(); 117 + }); 118 + 119 + const mockDid = "did:plc:test123"; 120 + const mockTime = Date.now() * 1000; 121 + const mockDisplayName = "Test User"; 122 + const mockDescription = "This is a test description"; 123 + 124 + describe("checkDescription", () => { 125 + describe("Global allow list", () => { 126 + it("should skip profiles from globally allowed DIDs", async () => { 127 + await checkDescription( 128 + "did:plc:globalallow", 129 + mockTime, 130 + mockDisplayName, 131 + mockDescription, 132 + ); 133 + 134 + expect(logger.warn).toHaveBeenCalledWith( 135 + { 136 + process: "CHECKDESCRIPTION", 137 + did: "did:plc:globalallow", 138 + time: mockTime, 139 + displayName: mockDisplayName, 140 + description: mockDescription, 141 + }, 142 + "Global AllowListed DID", 143 + ); 144 + expect(createAccountLabel).not.toHaveBeenCalled(); 145 + }); 146 + 147 + it("should process non-globally-allowed DIDs", async () => { 148 + await checkDescription( 149 + mockDid, 150 + mockTime, 151 + mockDisplayName, 152 + "spam content", 153 + ); 154 + 155 + expect(logger.warn).not.toHaveBeenCalledWith( 156 + expect.objectContaining({ did: mockDid }), 157 + "Global AllowListed DID", 158 + ); 159 + }); 160 + }); 161 + 162 + describe("Pattern matching and labeling", () => { 163 + it("should label profiles with matching descriptions", async () => { 164 + await checkDescription( 165 + mockDid, 166 + mockTime, 167 + mockDisplayName, 168 + "This is spam content", 169 + ); 170 + 171 + expect(logger.info).toHaveBeenCalledWith( 172 + { 173 + process: "CHECKDESCRIPTION", 174 + did: mockDid, 175 + time: mockTime, 176 + displayName: mockDisplayName, 177 + description: "This is spam content", 178 + label: "test-description", 179 + }, 180 + "Labeling account", 181 + ); 182 + expect(createAccountLabel).toHaveBeenCalledWith( 183 + mockDid, 184 + "test-description", 185 + expect.stringContaining("Test description check"), 186 + ); 187 + }); 188 + 189 + it("should not label profiles without matching descriptions", async () => { 190 + await checkDescription( 191 + mockDid, 192 + mockTime, 193 + mockDisplayName, 194 + "Normal content", 195 + ); 196 + 197 + expect(createAccountLabel).not.toHaveBeenCalledWith( 198 + mockDid, 199 + "test-description", 200 + expect.any(String), 201 + ); 202 + }); 203 + 204 + it("should not check description when description field is false", async () => { 205 + await checkDescription( 206 + mockDid, 207 + mockTime, 208 + "fake account", 209 + mockDescription, 210 + ); 211 + 212 + // test-displayname has displayName: true, description: false 213 + // so it should not trigger on description content 214 + expect(createAccountLabel).not.toHaveBeenCalledWith( 215 + mockDid, 216 + "test-displayname", 217 + expect.any(String), 218 + ); 219 + }); 220 + 221 + it("should handle empty description", async () => { 222 + await checkDescription(mockDid, mockTime, mockDisplayName, ""); 223 + 224 + expect(createAccountLabel).not.toHaveBeenCalled(); 225 + }); 226 + }); 227 + 228 + describe("Language filtering", () => { 229 + it("should check language-specific patterns for matching languages", async () => { 230 + vi.mocked(getLanguage).mockResolvedValue("eng"); 231 + 232 + await checkDescription( 233 + mockDid, 234 + mockTime, 235 + mockDisplayName, 236 + "hello world", 237 + ); 238 + 239 + expect(createAccountLabel).toHaveBeenCalledWith( 240 + mockDid, 241 + "language-specific", 242 + expect.any(String), 243 + ); 244 + }); 245 + 246 + it("should skip language-specific patterns for non-matching languages", async () => { 247 + vi.mocked(getLanguage).mockResolvedValue("spa"); 248 + 249 + await checkDescription( 250 + mockDid, 251 + mockTime, 252 + mockDisplayName, 253 + "hello world", 254 + ); 255 + 256 + expect(createAccountLabel).not.toHaveBeenCalledWith( 257 + mockDid, 258 + "language-specific", 259 + expect.any(String), 260 + ); 261 + }); 262 + }); 263 + 264 + describe("Whitelist handling", () => { 265 + it("should skip patterns when whitelist matches", async () => { 266 + await checkDescription( 267 + mockDid, 268 + mockTime, 269 + mockDisplayName, 270 + "this is good bad content", 271 + ); 272 + 273 + expect(logger.debug).toHaveBeenCalledWith( 274 + { 275 + process: "CHECKDESCRIPTION", 276 + did: mockDid, 277 + time: mockTime, 278 + displayName: mockDisplayName, 279 + description: "this is good bad content", 280 + }, 281 + "Whitelisted phrase found", 282 + ); 283 + expect(createAccountLabel).not.toHaveBeenCalledWith( 284 + mockDid, 285 + "whitelisted-test", 286 + expect.any(String), 287 + ); 288 + }); 289 + 290 + it("should label when pattern matches but whitelist doesn't", async () => { 291 + await checkDescription( 292 + mockDid, 293 + mockTime, 294 + mockDisplayName, 295 + "just bad content", 296 + ); 297 + 298 + expect(createAccountLabel).toHaveBeenCalledWith( 299 + mockDid, 300 + "whitelisted-test", 301 + expect.any(String), 302 + ); 303 + }); 304 + }); 305 + 306 + describe("Ignored DIDs", () => { 307 + it("should skip checks for ignored DIDs", async () => { 308 + await checkDescription( 309 + "did:plc:ignored123", 310 + mockTime, 311 + mockDisplayName, 312 + "test content", 313 + ); 314 + 315 + expect(logger.debug).toHaveBeenCalledWith( 316 + { 317 + process: "CHECKDESCRIPTION", 318 + did: "did:plc:ignored123", 319 + time: mockTime, 320 + displayName: mockDisplayName, 321 + description: "test content", 322 + }, 323 + "Whitelisted DID", 324 + ); 325 + expect(createAccountLabel).not.toHaveBeenCalledWith( 326 + "did:plc:ignored123", 327 + "ignored-did", 328 + expect.any(String), 329 + ); 330 + }); 331 + 332 + it("should check non-ignored DIDs", async () => { 333 + await checkDescription( 334 + mockDid, 335 + mockTime, 336 + mockDisplayName, 337 + "test content", 338 + ); 339 + 340 + expect(createAccountLabel).toHaveBeenCalledWith( 341 + mockDid, 342 + "ignored-did", 343 + expect.any(String), 344 + ); 345 + }); 346 + }); 347 + 348 + describe("Moderation actions", () => { 349 + it("should execute all moderation actions when enabled", async () => { 350 + await checkDescription( 351 + mockDid, 352 + mockTime, 353 + mockDisplayName, 354 + "report this content", 355 + ); 356 + 357 + expect(createAccountLabel).toHaveBeenCalledWith( 358 + mockDid, 359 + "all-actions", 360 + expect.any(String), 361 + ); 362 + expect(createAccountReport).toHaveBeenCalledWith( 363 + mockDid, 364 + expect.any(String), 365 + ); 366 + expect(createAccountComment).toHaveBeenCalledWith( 367 + mockDid, 368 + expect.any(String), 369 + ); 370 + }); 371 + 372 + it("should log all moderation actions", async () => { 373 + await checkDescription( 374 + mockDid, 375 + mockTime, 376 + mockDisplayName, 377 + "report this", 378 + ); 379 + 380 + expect(logger.info).toHaveBeenCalledWith( 381 + expect.objectContaining({ label: "all-actions" }), 382 + "Labeling account", 383 + ); 384 + expect(logger.info).toHaveBeenCalledWith( 385 + expect.objectContaining({ label: "all-actions" }), 386 + "Reporting account", 387 + ); 388 + expect(logger.info).toHaveBeenCalledWith( 389 + expect.objectContaining({ label: "all-actions" }), 390 + "Commenting on account", 391 + ); 392 + }); 393 + }); 394 + }); 395 + 396 + describe("checkDisplayName", () => { 397 + describe("Global allow list", () => { 398 + it("should skip profiles from globally allowed DIDs", async () => { 399 + await checkDisplayName( 400 + "did:plc:globalallow", 401 + mockTime, 402 + mockDisplayName, 403 + mockDescription, 404 + ); 405 + 406 + expect(logger.warn).toHaveBeenCalledWith( 407 + { 408 + process: "CHECKDISPLAYNAME", 409 + did: "did:plc:globalallow", 410 + time: mockTime, 411 + displayName: mockDisplayName, 412 + description: mockDescription, 413 + }, 414 + "Global AllowListed DID", 415 + ); 416 + expect(createAccountLabel).not.toHaveBeenCalled(); 417 + }); 418 + 419 + it("should process non-globally-allowed DIDs", async () => { 420 + await checkDisplayName(mockDid, mockTime, "fake user", mockDescription); 421 + 422 + expect(logger.warn).not.toHaveBeenCalledWith( 423 + expect.objectContaining({ did: mockDid }), 424 + "Global AllowListed DID", 425 + ); 426 + }); 427 + }); 428 + 429 + describe("Pattern matching and labeling", () => { 430 + it("should label profiles with matching display names", async () => { 431 + await checkDisplayName( 432 + mockDid, 433 + mockTime, 434 + "fake account", 435 + mockDescription, 436 + ); 437 + 438 + expect(logger.info).toHaveBeenCalledWith( 439 + { 440 + process: "CHECKDISPLAYNAME", 441 + did: mockDid, 442 + time: mockTime, 443 + displayName: "fake account", 444 + description: mockDescription, 445 + label: "test-displayname", 446 + }, 447 + "Labeling account", 448 + ); 449 + expect(createAccountLabel).toHaveBeenCalledWith( 450 + mockDid, 451 + "test-displayname", 452 + expect.stringContaining("Test display name check"), 453 + ); 454 + }); 455 + 456 + it("should not label profiles without matching display names", async () => { 457 + await checkDisplayName(mockDid, mockTime, "Normal User", mockDescription); 458 + 459 + expect(createAccountLabel).not.toHaveBeenCalledWith( 460 + mockDid, 461 + "test-displayname", 462 + expect.any(String), 463 + ); 464 + }); 465 + 466 + it("should not check displayName when displayName field is false", async () => { 467 + await checkDisplayName( 468 + mockDid, 469 + mockTime, 470 + mockDisplayName, 471 + "spam description", 472 + ); 473 + 474 + // test-description has description: true, displayName: false 475 + // so it should not trigger on displayName content 476 + expect(createAccountLabel).not.toHaveBeenCalledWith( 477 + mockDid, 478 + "test-description", 479 + expect.any(String), 480 + ); 481 + }); 482 + 483 + it("should handle empty display name", async () => { 484 + await checkDisplayName(mockDid, mockTime, "", mockDescription); 485 + 486 + expect(createAccountLabel).not.toHaveBeenCalled(); 487 + }); 488 + }); 489 + 490 + describe("Language filtering", () => { 491 + it("should check language-specific patterns for matching languages", async () => { 492 + vi.mocked(getLanguage).mockResolvedValue("eng"); 493 + 494 + await checkDisplayName( 495 + mockDid, 496 + mockTime, 497 + "hello world", 498 + "description", 499 + ); 500 + 501 + // language-specific check has description: true, displayName: false 502 + // so it won't match on displayName 503 + expect(createAccountLabel).not.toHaveBeenCalledWith( 504 + mockDid, 505 + "language-specific", 506 + expect.any(String), 507 + ); 508 + }); 509 + 510 + it("should skip language-specific patterns for non-matching languages", async () => { 511 + vi.mocked(getLanguage).mockResolvedValue("spa"); 512 + 513 + await checkDisplayName(mockDid, mockTime, "hello", mockDescription); 514 + 515 + expect(createAccountLabel).not.toHaveBeenCalledWith( 516 + mockDid, 517 + "language-specific", 518 + expect.any(String), 519 + ); 520 + }); 521 + }); 522 + 523 + describe("Whitelist handling", () => { 524 + it("should skip patterns when whitelist matches", async () => { 525 + await checkDisplayName( 526 + mockDid, 527 + mockTime, 528 + "good bad user", 529 + mockDescription, 530 + ); 531 + 532 + // whitelisted-test has description: true, displayName: false 533 + // so it won't trigger on displayName 534 + expect(createAccountLabel).not.toHaveBeenCalledWith( 535 + mockDid, 536 + "whitelisted-test", 537 + expect.any(String), 538 + ); 539 + }); 540 + }); 541 + 542 + describe("Ignored DIDs", () => { 543 + it("should skip checks for ignored DIDs", async () => { 544 + await checkDisplayName( 545 + "did:plc:ignored123", 546 + mockTime, 547 + "test user", 548 + mockDescription, 549 + ); 550 + 551 + expect(logger.debug).toHaveBeenCalledWith( 552 + { 553 + process: "CHECKDISPLAYNAME", 554 + did: "did:plc:ignored123", 555 + time: mockTime, 556 + displayName: "test user", 557 + description: mockDescription, 558 + }, 559 + "Whitelisted DID", 560 + ); 561 + expect(createAccountLabel).not.toHaveBeenCalledWith( 562 + "did:plc:ignored123", 563 + "ignored-did", 564 + expect.any(String), 565 + ); 566 + }); 567 + 568 + it("should check non-ignored DIDs", async () => { 569 + await checkDisplayName(mockDid, mockTime, "test user", mockDescription); 570 + 571 + // ignored-did has description: true, displayName: false 572 + // so it won't trigger on displayName 573 + expect(createAccountLabel).not.toHaveBeenCalledWith( 574 + mockDid, 575 + "ignored-did", 576 + expect.any(String), 577 + ); 578 + }); 579 + }); 580 + 581 + describe("Moderation actions", () => { 582 + it("should execute all moderation actions when enabled", async () => { 583 + await checkDisplayName( 584 + mockDid, 585 + mockTime, 586 + mockDisplayName, 587 + "report this content", 588 + ); 589 + 590 + // all-actions has description: true, displayName: false 591 + // so it won't match on displayName 592 + expect(createAccountLabel).not.toHaveBeenCalledWith( 593 + mockDid, 594 + "all-actions", 595 + expect.any(String), 596 + ); 597 + }); 598 + }); 599 + 600 + describe("Both fields check", () => { 601 + it("should check displayName when both fields are enabled", async () => { 602 + await checkDisplayName( 603 + mockDid, 604 + mockTime, 605 + "suspicious user", 606 + mockDescription, 607 + ); 608 + 609 + expect(createAccountLabel).toHaveBeenCalledWith( 610 + mockDid, 611 + "both-fields", 612 + expect.any(String), 613 + ); 614 + }); 615 + 616 + it("should check description when both fields are enabled", async () => { 617 + await checkDescription( 618 + mockDid, 619 + mockTime, 620 + mockDisplayName, 621 + "suspicious content", 622 + ); 623 + 624 + expect(createAccountLabel).toHaveBeenCalledWith( 625 + mockDid, 626 + "both-fields", 627 + expect.any(String), 628 + ); 629 + }); 630 + }); 631 + }); 632 + });
-1
src/types.ts
··· 68 68 comment: string; // Comment for the label 69 69 expires?: string; // Optional expiration date (ISO 8601) - check will be skipped after this date 70 70 } 71 -
-85
src/utils.ts
··· 1 - import { logger } from "./logger.js"; 2 - 3 - import { homoglyphMap } from "./homoglyphs.js"; 4 - 5 - /** 6 - * Normalizes a string by converting it to lowercase, replacing homoglyphs, 7 - * and stripping diacritics. This is useful for sanitizing user input 8 - * before performing checks for forbidden words. 9 - * 10 - * The process is as follows: 11 - * 1. Convert the entire string to lowercase. 12 - * 2. Replace characters that are visually similar to ASCII letters (homoglyphs) 13 - * with their ASCII counterparts based on the `homoglyphMap`. 14 - * 3. Apply NFD (Normalization Form D) Unicode normalization to decompose 15 - * characters into their base characters and combining marks. 16 - * 4. Remove all Unicode combining diacritical marks. 17 - * 5. Apply NFKC (Normalization Form KC) Unicode normalization for a final 18 - * cleanup, which handles compatibility characters. 19 - * 20 - * @param text The input string to normalize. 21 - * @returns The normalized string. 22 - */ 23 - export function normalizeUnicode(text: string): string { 24 - // Convert to lowercase to match the homoglyph map keys 25 - const lowercased = text.toLowerCase(); 26 - 27 - // Replace characters using the homoglyph map. 28 - // This is done before NFD so that pre-composed characters are caught. 29 - let replaced = ""; 30 - for (const char of lowercased) { 31 - replaced += homoglyphMap[char] || char; 32 - } 33 - 34 - // First decompose the characters (NFD), then remove diacritics. 35 - const withoutDiacritics = replaced 36 - .normalize("NFD") 37 - .replace(/[\u0300-\u036f]/g, ""); 38 - 39 - // Final NFKC normalization to handle any remaining special characters. 40 - return withoutDiacritics.normalize("NFKC"); 41 - } 42 - 43 - export async function getFinalUrl(url: string): Promise<string> { 44 - const controller = new AbortController(); 45 - const timeoutId = setTimeout(() => controller.abort(), 10000); // 10-second timeout 46 - 47 - try { 48 - const response = await fetch(url, { 49 - method: "HEAD", 50 - redirect: "follow", // This will follow redirects automatically 51 - signal: controller.signal, // Pass the abort signal to fetch 52 - }); 53 - clearTimeout(timeoutId); // Clear the timeout if fetch completes 54 - return response.url; // This will be the final URL after redirects 55 - } catch (error) { 56 - clearTimeout(timeoutId); // Clear the timeout if fetch fails 57 - // Log the error with more specific information if it's a timeout 58 - if (error instanceof Error && error.name === "AbortError") { 59 - logger.warn({ process: "UTILS", url, error }, "Timeout fetching URL"); 60 - } else { 61 - logger.warn({ process: "UTILS", url, error }, "Error fetching URL"); 62 - } 63 - throw error; // Re-throw the error to be caught by the caller 64 - } 65 - } 66 - 67 - export async function getLanguage(profile: string): Promise<string> { 68 - if (typeof profile !== "string") { 69 - logger.warn({ process: "UTILS", profile }, "getLanguage called with invalid profile data, defaulting to 'eng'"); 70 - return "eng"; // Default or throw an error 71 - } 72 - 73 - const profileText = profile.trim(); 74 - 75 - if (profileText.length === 0) { 76 - return "eng"; 77 - } 78 - 79 - const { franc } = await import("franc"); 80 - const detectedLang = franc(profileText); 81 - 82 - // franc returns "und" (undetermined) if it can't detect the language 83 - // Default to "eng" in such cases 84 - return detectedLang === "und" ? "eng" : detectedLang; 85 - }
+256
src/utils/getFinalUrl.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { getFinalUrl } from "./getFinalUrl.js"; 3 + 4 + // Mock the logger 5 + vi.mock("../logger.js", () => ({ 6 + logger: { 7 + debug: vi.fn(), 8 + warn: vi.fn(), 9 + }, 10 + })); 11 + 12 + describe("getFinalUrl", () => { 13 + beforeEach(() => { 14 + vi.clearAllMocks(); 15 + vi.unstubAllGlobals(); 16 + }); 17 + 18 + describe("successful HEAD requests", () => { 19 + it("should return the final URL after redirect", async () => { 20 + const mockFetch = vi.fn().mockResolvedValue({ 21 + url: "https://example.com/final", 22 + }); 23 + vi.stubGlobal("fetch", mockFetch); 24 + 25 + const result = await getFinalUrl("https://example.com/redirect"); 26 + 27 + expect(result).toBe("https://example.com/final"); 28 + expect(mockFetch).toHaveBeenCalledWith( 29 + "https://example.com/redirect", 30 + expect.objectContaining({ 31 + method: "HEAD", 32 + redirect: "follow", 33 + }), 34 + ); 35 + }); 36 + 37 + it("should return the same URL if no redirect", async () => { 38 + const mockFetch = vi.fn().mockResolvedValue({ 39 + url: "https://example.com/page", 40 + }); 41 + vi.stubGlobal("fetch", mockFetch); 42 + 43 + const result = await getFinalUrl("https://example.com/page"); 44 + 45 + expect(result).toBe("https://example.com/page"); 46 + }); 47 + 48 + it("should include proper user agent header", async () => { 49 + const mockFetch = vi.fn().mockResolvedValue({ 50 + url: "https://example.com/final", 51 + }); 52 + vi.stubGlobal("fetch", mockFetch); 53 + 54 + await getFinalUrl("https://example.com/test"); 55 + 56 + expect(mockFetch).toHaveBeenCalledWith( 57 + "https://example.com/test", 58 + expect.objectContaining({ 59 + headers: { 60 + "User-Agent": expect.stringContaining("SkyWatch"), 61 + }, 62 + }), 63 + ); 64 + }); 65 + }); 66 + 67 + describe("HEAD request failures with GET fallback", () => { 68 + it("should fallback to GET when HEAD fails", async () => { 69 + const mockFetch = vi 70 + .fn() 71 + .mockRejectedValueOnce(new Error("HEAD not allowed")) 72 + .mockResolvedValueOnce({ 73 + url: "https://example.com/final", 74 + }); 75 + vi.stubGlobal("fetch", mockFetch); 76 + 77 + const result = await getFinalUrl("https://example.com/test"); 78 + 79 + expect(result).toBe("https://example.com/final"); 80 + expect(mockFetch).toHaveBeenCalledTimes(2); 81 + expect(mockFetch).toHaveBeenNthCalledWith( 82 + 1, 83 + "https://example.com/test", 84 + expect.objectContaining({ method: "HEAD" }), 85 + ); 86 + expect(mockFetch).toHaveBeenNthCalledWith( 87 + 2, 88 + "https://example.com/test", 89 + expect.objectContaining({ method: "GET" }), 90 + ); 91 + }); 92 + 93 + it("should handle network errors on HEAD with GET success", async () => { 94 + const mockFetch = vi 95 + .fn() 96 + .mockRejectedValueOnce(new Error("Network error")) 97 + .mockResolvedValueOnce({ 98 + url: "https://example.com/final", 99 + }); 100 + vi.stubGlobal("fetch", mockFetch); 101 + 102 + const result = await getFinalUrl("https://example.com/test"); 103 + 104 + expect(result).toBe("https://example.com/final"); 105 + expect(mockFetch).toHaveBeenCalledTimes(2); 106 + }); 107 + }); 108 + 109 + describe("timeout handling", () => { 110 + it("should configure abort signal with timeout for HEAD request", async () => { 111 + const mockFetch = vi.fn().mockResolvedValue({ 112 + url: "https://example.com/final", 113 + }); 114 + vi.stubGlobal("fetch", mockFetch); 115 + 116 + await getFinalUrl("https://example.com/test"); 117 + 118 + expect(mockFetch).toHaveBeenCalledWith( 119 + "https://example.com/test", 120 + expect.objectContaining({ 121 + signal: expect.any(AbortSignal), 122 + }), 123 + ); 124 + }); 125 + 126 + it("should handle AbortError from timeout", async () => { 127 + const mockFetch = vi.fn().mockRejectedValue( 128 + Object.assign(new Error("The operation was aborted"), { 129 + name: "AbortError", 130 + }), 131 + ); 132 + vi.stubGlobal("fetch", mockFetch); 133 + 134 + await expect(getFinalUrl("https://slow.example.com")).rejects.toThrow( 135 + "The operation was aborted", 136 + ); 137 + }); 138 + }); 139 + 140 + describe("complete failure scenarios", () => { 141 + it("should throw error when both HEAD and GET fail", async () => { 142 + const mockFetch = vi.fn().mockRejectedValue(new Error("Network error")); 143 + vi.stubGlobal("fetch", mockFetch); 144 + 145 + await expect(getFinalUrl("https://example.com/test")).rejects.toThrow( 146 + "Network error", 147 + ); 148 + expect(mockFetch).toHaveBeenCalledTimes(2); 149 + }); 150 + 151 + it("should throw AbortError on timeout", async () => { 152 + const mockFetch = vi.fn().mockImplementation(() => { 153 + const error = new Error("The operation was aborted"); 154 + error.name = "AbortError"; 155 + return Promise.reject(error); 156 + }); 157 + vi.stubGlobal("fetch", mockFetch); 158 + 159 + await expect(getFinalUrl("https://example.com/test")).rejects.toThrow(); 160 + }); 161 + 162 + it("should handle non-Error exceptions", async () => { 163 + const mockFetch = vi.fn().mockRejectedValue("string error"); 164 + vi.stubGlobal("fetch", mockFetch); 165 + 166 + await expect(getFinalUrl("https://example.com/test")).rejects.toBe( 167 + "string error", 168 + ); 169 + }); 170 + }); 171 + 172 + describe("URL redirect chains", () => { 173 + it("should handle multiple redirects", async () => { 174 + const mockFetch = vi.fn().mockResolvedValue({ 175 + url: "https://example.com/final-destination", 176 + }); 177 + vi.stubGlobal("fetch", mockFetch); 178 + 179 + const result = await getFinalUrl("https://t.co/shortlink"); 180 + 181 + expect(result).toBe("https://example.com/final-destination"); 182 + }); 183 + 184 + it("should preserve query parameters in final URL", async () => { 185 + const mockFetch = vi.fn().mockResolvedValue({ 186 + url: "https://example.com/page?param=value&other=test", 187 + }); 188 + vi.stubGlobal("fetch", mockFetch); 189 + 190 + const result = await getFinalUrl("https://example.com/redirect"); 191 + 192 + expect(result).toBe("https://example.com/page?param=value&other=test"); 193 + }); 194 + 195 + it("should handle fragment identifiers", async () => { 196 + const mockFetch = vi.fn().mockResolvedValue({ 197 + url: "https://example.com/page#section", 198 + }); 199 + vi.stubGlobal("fetch", mockFetch); 200 + 201 + const result = await getFinalUrl("https://example.com/redirect"); 202 + 203 + expect(result).toBe("https://example.com/page#section"); 204 + }); 205 + }); 206 + 207 + describe("edge cases", () => { 208 + it("should handle URLs with special characters", async () => { 209 + const mockFetch = vi.fn().mockResolvedValue({ 210 + url: "https://example.com/page?query=hello%20world", 211 + }); 212 + vi.stubGlobal("fetch", mockFetch); 213 + 214 + const result = await getFinalUrl( 215 + "https://example.com/page?query=hello%20world", 216 + ); 217 + 218 + expect(result).toBe("https://example.com/page?query=hello%20world"); 219 + }); 220 + 221 + it("should handle internationalized domain names", async () => { 222 + const mockFetch = vi.fn().mockResolvedValue({ 223 + url: "https://xn--example-r63b.com/", 224 + }); 225 + vi.stubGlobal("fetch", mockFetch); 226 + 227 + const result = await getFinalUrl("https://example.com/redirect"); 228 + 229 + expect(result).toBe("https://xn--example-r63b.com/"); 230 + }); 231 + }); 232 + 233 + describe("error serialization", () => { 234 + it("should properly serialize Error objects", async () => { 235 + const error = new Error("Test error"); 236 + error.cause = "underlying cause"; 237 + 238 + const mockFetch = vi.fn().mockRejectedValue(error); 239 + vi.stubGlobal("fetch", mockFetch); 240 + 241 + await expect(getFinalUrl("https://example.com/test")).rejects.toThrow( 242 + "Test error", 243 + ); 244 + }); 245 + 246 + it("should handle errors without cause", async () => { 247 + const error = new Error("Simple error"); 248 + const mockFetch = vi.fn().mockRejectedValue(error); 249 + vi.stubGlobal("fetch", mockFetch); 250 + 251 + await expect(getFinalUrl("https://example.com/test")).rejects.toThrow( 252 + "Simple error", 253 + ); 254 + }); 255 + }); 256 + });
+70
src/utils/getFinalUrl.ts
··· 1 + import { logger } from "../logger.js"; 2 + 3 + export async function getFinalUrl(url: string): Promise<string> { 4 + const controller = new AbortController(); 5 + const timeoutId = setTimeout(() => controller.abort(), 15000); // 15-second timeout 6 + 7 + const headers = { 8 + "User-Agent": 9 + "Mozilla/5.0 (compatible; SkyWatch/1.0; +https://github.com/skywatch-bsky/skywatch-automod)", 10 + }; 11 + 12 + try { 13 + // Try HEAD request first (faster, less bandwidth) 14 + const response = await fetch(url, { 15 + method: "HEAD", 16 + redirect: "follow", 17 + signal: controller.signal, 18 + headers, 19 + }); 20 + clearTimeout(timeoutId); 21 + return response.url; 22 + } catch (headError) { 23 + clearTimeout(timeoutId); 24 + 25 + // Some services block HEAD requests, try GET as fallback 26 + const getController = new AbortController(); 27 + const getTimeoutId = setTimeout(() => getController.abort(), 15000); 28 + 29 + try { 30 + logger.debug( 31 + { process: "UTILS", url, method: "HEAD" }, 32 + "HEAD request failed, trying GET", 33 + ); 34 + 35 + const response = await fetch(url, { 36 + method: "GET", 37 + redirect: "follow", 38 + signal: getController.signal, 39 + headers, 40 + }); 41 + clearTimeout(getTimeoutId); 42 + return response.url; 43 + } catch (error) { 44 + clearTimeout(getTimeoutId); 45 + 46 + // Properly serialize error information 47 + const errorInfo = 48 + error instanceof Error 49 + ? { 50 + name: error.name, 51 + message: error.message, 52 + cause: error.cause, 53 + } 54 + : { error: String(error) }; 55 + 56 + if (error instanceof Error && error.name === "AbortError") { 57 + logger.warn( 58 + { process: "UTILS", url, ...errorInfo }, 59 + "Timeout resolving URL", 60 + ); 61 + } else { 62 + logger.warn( 63 + { process: "UTILS", url, ...errorInfo }, 64 + "Failed to resolve URL", 65 + ); 66 + } 67 + throw error; 68 + } 69 + } 70 + }
+157
src/utils/getLanguage.test.ts
··· 1 + import { describe, it, expect, vi, beforeEach } from "vitest"; 2 + import { getLanguage } from "./getLanguage.js"; 3 + 4 + // Mock the logger 5 + vi.mock("../logger.js", () => ({ 6 + logger: { 7 + warn: vi.fn(), 8 + }, 9 + })); 10 + 11 + describe("getLanguage", () => { 12 + beforeEach(() => { 13 + vi.clearAllMocks(); 14 + }); 15 + 16 + describe("language detection", () => { 17 + it("should detect English text", async () => { 18 + const text = "Hello world, this is a test of the English language."; 19 + const result = await getLanguage(text); 20 + expect(result).toBe("eng"); 21 + }); 22 + 23 + it("should detect Spanish text", async () => { 24 + const text = 25 + "Hola mundo, esta es una prueba del idioma español con suficiente texto para detectar."; 26 + const result = await getLanguage(text); 27 + expect(result).toBe("spa"); 28 + }); 29 + 30 + it("should detect French text", async () => { 31 + const text = 32 + "Bonjour le monde, ceci est un test de la langue française avec suffisamment de texte."; 33 + const result = await getLanguage(text); 34 + expect(result).toBe("fra"); 35 + }); 36 + 37 + it("should detect German text", async () => { 38 + const text = 39 + "Hallo Welt, dies ist ein Test der deutschen Sprache mit genügend Text."; 40 + const result = await getLanguage(text); 41 + expect(result).toBe("deu"); 42 + }); 43 + 44 + it("should detect Portuguese text", async () => { 45 + const text = 46 + "Olá mundo, este é um teste da língua portuguesa com texto suficiente para detecção."; 47 + const result = await getLanguage(text); 48 + expect(result).toBe("por"); 49 + }); 50 + 51 + it("should detect Italian text", async () => { 52 + const text = 53 + "Ciao mondo, questo è un test della lingua italiana con abbastanza testo."; 54 + const result = await getLanguage(text); 55 + expect(result).toBe("ita"); 56 + }); 57 + 58 + it("should detect Japanese text", async () => { 59 + const text = "これは日本語のテストです。十分なテキストで言語を検出します。"; 60 + const result = await getLanguage(text); 61 + expect(result).toBe("jpn"); 62 + }); 63 + }); 64 + 65 + describe("edge cases", () => { 66 + it("should default to eng for empty strings", async () => { 67 + const result = await getLanguage(""); 68 + expect(result).toBe("eng"); 69 + }); 70 + 71 + it("should default to eng for whitespace-only strings", async () => { 72 + const result = await getLanguage(" "); 73 + expect(result).toBe("eng"); 74 + }); 75 + 76 + it("should default to eng for very short text", async () => { 77 + const result = await getLanguage("hi"); 78 + expect(result).toBe("eng"); 79 + }); 80 + 81 + it("should default to eng for undetermined language", async () => { 82 + const result = await getLanguage("123 456 789"); 83 + expect(result).toBe("eng"); 84 + }); 85 + 86 + it("should default to eng for symbols only", async () => { 87 + const result = await getLanguage("!@#$%^&*()"); 88 + expect(result).toBe("eng"); 89 + }); 90 + }); 91 + 92 + describe("invalid input handling", () => { 93 + it("should handle non-string input gracefully", async () => { 94 + const result = await getLanguage(123 as any); 95 + expect(result).toBe("eng"); 96 + }); 97 + 98 + it("should handle null input gracefully", async () => { 99 + const result = await getLanguage(null as any); 100 + expect(result).toBe("eng"); 101 + }); 102 + 103 + it("should handle undefined input gracefully", async () => { 104 + const result = await getLanguage(undefined as any); 105 + expect(result).toBe("eng"); 106 + }); 107 + 108 + it("should handle object input gracefully", async () => { 109 + const result = await getLanguage({} as any); 110 + expect(result).toBe("eng"); 111 + }); 112 + 113 + it("should handle array input gracefully", async () => { 114 + const result = await getLanguage([] as any); 115 + expect(result).toBe("eng"); 116 + }); 117 + }); 118 + 119 + describe("trimming behavior", () => { 120 + it("should trim leading whitespace", async () => { 121 + const text = 122 + " Hello world, this is a test of the English language."; 123 + const result = await getLanguage(text); 124 + expect(result).toBe("eng"); 125 + }); 126 + 127 + it("should trim trailing whitespace", async () => { 128 + const text = 129 + "Hello world, this is a test of the English language. "; 130 + const result = await getLanguage(text); 131 + expect(result).toBe("eng"); 132 + }); 133 + 134 + it("should trim both leading and trailing whitespace", async () => { 135 + const text = 136 + " Hello world, this is a test of the English language. "; 137 + const result = await getLanguage(text); 138 + expect(result).toBe("eng"); 139 + }); 140 + }); 141 + 142 + describe("mixed language text", () => { 143 + it("should detect primary language in mixed content", async () => { 144 + const text = 145 + "This is primarily English text with some español words mixed in."; 146 + const result = await getLanguage(text); 147 + expect(result).toBe("eng"); 148 + }); 149 + 150 + it("should handle code mixed with text", async () => { 151 + const text = 152 + "Here is some English text with const x = 123; code in it."; 153 + const result = await getLanguage(text); 154 + expect(result).toBe("eng"); 155 + }); 156 + }); 157 + });
+24
src/utils/getLanguage.ts
··· 1 + import { logger } from "../logger.js"; 2 + 3 + export async function getLanguage(profile: string): Promise<string> { 4 + if (typeof profile !== "string") { 5 + logger.warn( 6 + { process: "UTILS", profile }, 7 + "getLanguage called with invalid profile data, defaulting to 'eng'", 8 + ); 9 + return "eng"; // Default or throw an error 10 + } 11 + 12 + const profileText = profile.trim(); 13 + 14 + if (profileText.length === 0) { 15 + return "eng"; 16 + } 17 + 18 + const { franc } = await import("franc"); 19 + const detectedLang = franc(profileText); 20 + 21 + // franc returns "und" (undetermined) if it can't detect the language 22 + // Default to "eng" in such cases 23 + return detectedLang === "und" ? "eng" : detectedLang; 24 + }
+99
src/utils/normalizeUnicode.test.ts
··· 1 + import { describe, it, expect } from "vitest"; 2 + import { normalizeUnicode } from "./normalizeUnicode.js"; 3 + 4 + describe("normalizeUnicode", () => { 5 + describe("lowercase conversion", () => { 6 + it("should convert uppercase to lowercase", () => { 7 + expect(normalizeUnicode("HELLO")).toBe("hello"); 8 + expect(normalizeUnicode("WoRlD")).toBe("world"); 9 + }); 10 + }); 11 + 12 + describe("homoglyph replacement", () => { 13 + it("should replace homoglyphs with ASCII equivalents", () => { 14 + expect(normalizeUnicode("h3ll0")).toBe("hello"); 15 + expect(normalizeUnicode("t3st")).toBe("test"); 16 + expect(normalizeUnicode("@ppl3")).toBe("apple"); 17 + }); 18 + 19 + it("should replace accented characters", () => { 20 + expect(normalizeUnicode("café")).toBe("cafe"); 21 + expect(normalizeUnicode("naïve")).toBe("naive"); 22 + expect(normalizeUnicode("résumé")).toBe("resume"); 23 + }); 24 + 25 + it("should handle cyrillic lookalikes", () => { 26 + expect(normalizeUnicode("tеst")).toBe("test"); // е is cyrillic 27 + expect(normalizeUnicode("пight")).toBe("night"); // п is cyrillic 28 + }); 29 + }); 30 + 31 + describe("diacritic removal", () => { 32 + it("should remove combining diacritical marks", () => { 33 + expect(normalizeUnicode("e\u0301")).toBe("e"); // e with combining acute accent 34 + expect(normalizeUnicode("a\u0300")).toBe("a"); // a with combining grave accent 35 + }); 36 + 37 + it("should handle precomposed characters", () => { 38 + expect(normalizeUnicode("é")).toBe("e"); 39 + expect(normalizeUnicode("ñ")).toBe("n"); 40 + expect(normalizeUnicode("ü")).toBe("u"); 41 + }); 42 + }); 43 + 44 + describe("unicode normalization", () => { 45 + it("should normalize compatibility characters", () => { 46 + expect(normalizeUnicode("fi")).toBe("fi"); // ligature fi 47 + expect(normalizeUnicode("a")).toBe("a"); // fullwidth a 48 + }); 49 + 50 + it("should handle complex unicode sequences", () => { 51 + const input = "Ḧëḷḷö Ẅöṛḷḋ"; 52 + const expected = "hello world"; 53 + expect(normalizeUnicode(input)).toBe(expected); 54 + }); 55 + }); 56 + 57 + describe("edge cases", () => { 58 + it("should handle empty strings", () => { 59 + expect(normalizeUnicode("")).toBe(""); 60 + }); 61 + 62 + it("should handle strings with only spaces", () => { 63 + expect(normalizeUnicode(" ")).toBe(" "); 64 + }); 65 + 66 + it("should preserve non-mapped characters", () => { 67 + expect(normalizeUnicode("hello!@#$%")).toBe("hello!a#$%"); // @ maps to a 68 + }); 69 + 70 + it("should handle mixed scripts", () => { 71 + const input = "hëllö wörld"; 72 + expect(normalizeUnicode(input)).toBe("hello world"); 73 + }); 74 + 75 + it("should be idempotent", () => { 76 + const input = "tést"; 77 + const normalized = normalizeUnicode(input); 78 + expect(normalizeUnicode(normalized)).toBe(normalized); 79 + }); 80 + }); 81 + 82 + describe("real-world examples", () => { 83 + it("should normalize common slur evasion techniques", () => { 84 + expect(normalizeUnicode("f@gg0t")).toBe("faggot"); 85 + expect(normalizeUnicode("n1gg3r")).toBe("nigger"); 86 + expect(normalizeUnicode("k1k3")).toBe("kike"); 87 + }); 88 + 89 + it("should normalize unicode evasion techniques", () => { 90 + expect(normalizeUnicode("fаggоt")).toBe("faggot"); // cyrillic а and о 91 + expect(normalizeUnicode("nіggеr")).toBe("nigger"); // cyrillic і and е 92 + }); 93 + 94 + it("should handle multiple evasion techniques combined", () => { 95 + expect(normalizeUnicode("F@GG0Т")).toBe("faggot"); // mixed case, numbers, cyrillic 96 + expect(normalizeUnicode("n1ggёr")).toBe("nigger"); // numbers and cyrillic 97 + }); 98 + }); 99 + });
+43
src/utils/normalizeUnicode.ts
··· 1 + import { logger } from "../logger.js"; 2 + 3 + import { homoglyphMap } from "./homoglyphs.js"; 4 + 5 + /** 6 + * Normalizes a string by converting it to lowercase, replacing homoglyphs, 7 + * and stripping diacritics. This is useful for sanitizing user input 8 + * before performing checks for forbidden words. 9 + * 10 + * The process is as follows: 11 + * 1. Convert the entire string to lowercase. 12 + * 2. Replace characters that are visually similar to ASCII letters (homoglyphs) 13 + * with their ASCII counterparts based on the `homoglyphMap`. 14 + * 3. Apply NFD (Normalization Form D) Unicode normalization to decompose 15 + * characters into their base characters and combining marks. 16 + * 4. Remove all Unicode combining diacritical marks. 17 + * 5. Apply NFKC (Normalization Form KC) Unicode normalization for a final 18 + * cleanup, which handles compatibility characters. 19 + * 20 + * @param text The input string to normalize. 21 + * @returns The normalized string. 22 + */ 23 + export function normalizeUnicode(text: string): string { 24 + // Convert to lowercase to match the homoglyph map keys 25 + const lowercased = text.toLowerCase(); 26 + 27 + // Replace characters using the homoglyph map. 28 + // This is done before NFD so that pre-composed characters are caught. 29 + let replaced = ""; 30 + for (const char of lowercased) { 31 + replaced += homoglyphMap[char] || char; 32 + } 33 + 34 + // First decompose the characters (NFD), then remove diacritics. 35 + const withoutDiacritics = replaced 36 + .normalize("NFD") 37 + .replace(/[\u0300-\u036f]/g, ""); 38 + 39 + // Final NFKC normalization to handle any remaining special characters. 40 + return withoutDiacritics.normalize("NFKC"); 41 + } 42 + 43 +
+13
vitest.config.ts
··· 7 7 coverage: { 8 8 provider: "v8", 9 9 reporter: ["text", "json", "html"], 10 + exclude: [ 11 + "node_modules/**", 12 + "dist/**", 13 + "**/*.config.*", 14 + "**/main.ts", 15 + "**/*.test.ts", 16 + ], 17 + thresholds: { 18 + lines: 60, 19 + functions: 60, 20 + branches: 60, 21 + statements: 60, 22 + }, 10 23 }, 11 24 }, 12 25 });