skywatch.blue
A tool for parsing traffic on the jetstream and applying a moderation workstream based on regexp based rules
Updates from private repo to add functionality for unpacking shortened links.
+79
-23
+22
-22
package.json
+22
-22
package.json
···
1
1
{
2
-
"name": "skywatch-automod",
3
-
"version": "1.0.0",
2
+
"name": "skywatch-tools",
3
+
"version": "1.1.0",
4
4
"type": "module",
5
5
"scripts": {
6
6
"start": "npx tsx src/main.ts",
···
14
14
"*": "prettier --ignore-unknown --write"
15
15
},
16
16
"devDependencies": {
17
-
"@eslint/js": "^9.15.0",
17
+
"@eslint/js": "^9.25.1",
18
18
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
19
-
"@types/better-sqlite3": "^7.6.12",
19
+
"@types/better-sqlite3": "^7.6.13",
20
20
"@types/eslint__js": "^8.42.3",
21
21
"@types/express": "^4.17.21",
22
-
"@types/node": "^22.9.1",
23
-
"eslint": "^9.15.0",
24
-
"prettier": "^3.3.3",
25
-
"tsx": "^4.19.2",
26
-
"typescript": "^5.6.3",
27
-
"typescript-eslint": "^8.15.0"
22
+
"@types/node": "^22.15.3",
23
+
"eslint": "^9.25.1",
24
+
"prettier": "^3.5.3",
25
+
"tsx": "^4.19.4",
26
+
"typescript": "^5.8.3",
27
+
"typescript-eslint": "^8.31.1"
28
28
},
29
29
"dependencies": {
30
-
"@atproto/api": "^0.13.23",
30
+
"@atproto/api": "^0.13.35",
31
31
"@atproto/bsky": "^0.0.101",
32
-
"@atproto/lexicon": "^0.4.4",
33
-
"@atproto/ozone": "^0.1.62",
34
-
"@atproto/repo": "^0.6.0",
35
-
"@atproto/xrpc-server": "^0.7.4",
36
-
"@skyware/bot": "^0.3.7",
37
-
"@skyware/jetstream": "^0.2.0",
32
+
"@atproto/lexicon": "^0.4.10",
33
+
"@atproto/ozone": "^0.1.108",
34
+
"@atproto/repo": "^0.6.5",
35
+
"@atproto/xrpc-server": "^0.7.17",
36
+
"@skyware/bot": "^0.3.11",
37
+
"@skyware/jetstream": "^0.2.2",
38
38
"@skyware/labeler": "^0.1.13",
39
39
"bottleneck": "^2.19.5",
40
-
"dotenv": "^16.4.5",
41
-
"express": "^4.21.1",
40
+
"dotenv": "^16.5.0",
41
+
"express": "^4.21.2",
42
42
"husky": "^9.1.7",
43
-
"lint-staged": "^15.2.10",
43
+
"lint-staged": "^15.5.1",
44
44
"p-ratelimit": "^1.0.1",
45
-
"pino": "^9.5.0",
45
+
"pino": "^9.6.0",
46
46
"pino-pretty": "^13.0.0",
47
47
"prom-client": "^15.1.3",
48
-
"undici": "^7.2.0"
48
+
"undici": "^7.8.0"
49
49
}
50
50
}
+35
src/checkPosts.ts
+35
src/checkPosts.ts
···
5
5
createPostLabel,
6
6
createAccountReport,
7
7
createAccountComment,
8
+
createPostReport,
8
9
} from "./moderation.js";
10
+
import { LINK_SHORTENER } from "./constants.js";
11
+
import { getFinalUrl } from "./utils.js";
9
12
10
13
export const checkPosts = async (post: Post[]) => {
11
14
// Get a list of labels
···
14
17
(postCheck) => postCheck.label,
15
18
);
16
19
20
+
const urlRegex = /https?:\/\/[^\s]+/g;
21
+
22
+
// Check for link shorteners
23
+
if (LINK_SHORTENER.test(post[0].text)) {
24
+
try {
25
+
const url = post[0].text.match(urlRegex);
26
+
if (url) {
27
+
const finalUrl = await getFinalUrl(url[0]);
28
+
if (finalUrl) {
29
+
const originalUrl = post[0].text;
30
+
post[0].text = finalUrl;
31
+
logger.info(`Shortened URL resolved: ${originalUrl} -> ${finalUrl}`);
32
+
}
33
+
}
34
+
} catch (error) {
35
+
logger.error(`Failed to resolve shortened URL: ${post[0].text}`, error);
36
+
// Keep the original URL if resolution fails
37
+
}
38
+
}
39
+
17
40
// iterate through the labels
18
41
labels.forEach((label) => {
19
42
const checkPost = POST_CHECKS.find(
···
42
65
post[0].atURI,
43
66
post[0].cid,
44
67
`${checkPost!.label}`,
68
+
`${post[0].time}: ${checkPost!.comment} at ${post[0].atURI} with text "${post[0].text}"`,
69
+
);
70
+
}
71
+
72
+
if (checkPost!.reportPost === true) {
73
+
logger.info(
74
+
`Suspected ${checkPost!.label} in post at ${post[0].atURI}`,
75
+
);
76
+
logger.info(`Reporting: ${post[0].atURI}`);
77
+
createPostReport(
78
+
post[0].atURI,
79
+
post[0].cid,
45
80
`${post[0].time}: ${checkPost!.comment} at ${post[0].atURI} with text "${post[0].text}"`,
46
81
);
47
82
}
+7
-1
src/constants.ts.example
+7
-1
src/constants.ts.example
···
1
+
// rename this file to constants.ts
2
+
3
+
1
4
import { Checks } from "./types.js";
2
5
3
-
// rename this to constants.ts
6
+
export const LINK_SHORTENER = new RegExp(
7
+
"(?:https?:\\/\\/)?([^.]+\\.)?(tinyurl\\.com|bit\\.ly|goo\\.gl|g\\.co|ow\\.ly|shorturl\\.at|t\\.co)",
8
+
"i",
9
+
);
4
10
5
11
export const PROFILE_CHECKS: Checks[] = [
6
12
{
+1
src/types.ts
+1
src/types.ts
+14
src/utils.ts
+14
src/utils.ts
···
31
31
// Final NFKC normalization to handle any remaining special characters
32
32
return withoutMath.normalize("NFKC");
33
33
}
34
+
35
+
export async function getFinalUrl(url: string): Promise<string> {
36
+
try {
37
+
const response = await fetch(url, {
38
+
method: "HEAD",
39
+
redirect: "follow", // This will follow redirects automatically
40
+
});
41
+
42
+
return response.url; // This will be the final URL after redirects
43
+
} catch (error) {
44
+
console.error("Error fetching URL:", error);
45
+
throw error;
46
+
}
47
+
}