gearsco.de
An Erlang lexer and syntax highlighter in Gleam
Add tests for errors
+156
-20
+10
-14
src/pearl.gleam
+10
-14
src/pearl.gleam
···
32
32
InvalidRadix(radix: String)
33
33
NumericSeparatorNotAllowed
34
34
ExpectedExponent
35
-
NumbersCannotEndAfterRadix
35
+
NumberCannotEndAfterRadix
36
36
UnterminatedCharacter
37
37
UnterminatedEscapeSequence
38
38
ExpectedSigilDelimiter
···
73
73
74
74
pub fn tokenise(lexer: Lexer) -> #(List(Token), List(Error)) {
75
75
do_tokenise(lexer, [])
76
+
}
77
+
78
+
pub fn to_source(tokens: List(Token)) -> String {
79
+
list.fold(tokens, "", fn(code, token) { code <> token.to_source(token) })
76
80
}
77
81
78
82
fn do_tokenise(lexer: Lexer, tokens: List(Token)) -> #(List(Token), List(Error)) {
···
563
567
token.Integer(string.drop_end(lexed, 1)),
564
568
)
565
569
AfterExponent -> #(error(lexer, ExpectedExponent), token)
566
-
AfterRadix -> #(error(lexer, NumbersCannotEndAfterRadix), token)
570
+
AfterRadix -> #(error(lexer, NumberCannotEndAfterRadix), token)
567
571
AfterNumber -> #(lexer, token)
568
572
AfterSeparator -> #(error(lexer, NumericSeparatorNotAllowed), token)
569
573
}
···
684
688
token.UnterminatedString(contents <> before),
685
689
)
686
690
687
-
"\\" ->
688
-
case string.pop_grapheme(after) {
689
-
Error(_) -> #(
690
-
error(advance(lexer, after), UnterminatedString),
691
-
token.UnterminatedString(contents),
692
-
)
693
-
Ok(#(character, source)) ->
694
-
lex_string(
695
-
advance(lexer, source),
696
-
contents <> before <> "\\" <> character,
697
-
)
698
-
}
691
+
"\\" -> {
692
+
let #(lexer, escape) = lex_escape_sequence(advance(lexer, after))
693
+
lex_string(lexer, contents <> before <> "\\" <> escape)
694
+
}
699
695
700
696
_ -> #(advance(lexer, after), token.String(contents <> before))
701
697
}
+146
-6
test/pearl_test.gleam
+146
-6
test/pearl_test.gleam
···
1
+
import gleam/list
1
2
import gleeunit
2
-
import gleeunit/should
3
+
import pearl
4
+
import pearl/token
5
+
import simplifile
3
6
4
-
pub fn main() -> Nil {
7
+
pub fn main() {
5
8
gleeunit.main()
6
9
}
7
10
8
-
// gleeunit test functions end in `_test`
9
-
pub fn hello_world_test() {
10
-
1
11
-
|> should.equal(1)
11
+
fn assert_roundtrip(src: String, allow_errors: Bool) -> Nil {
12
+
let #(tokens, errors) = src |> pearl.new |> pearl.tokenise
13
+
case allow_errors {
14
+
True -> Nil
15
+
False -> {
16
+
assert errors == []
17
+
}
18
+
}
19
+
20
+
assert pearl.to_source(tokens) == src
21
+
}
22
+
23
+
fn assert_tokens(src: String, tokens: List(token.Token)) -> Nil {
24
+
let #(lexed, errors) =
25
+
src |> pearl.new |> pearl.ignore_whitespace |> pearl.tokenise
26
+
assert errors == []
27
+
let tokens = list.append(tokens, [token.EndOfFile])
28
+
assert lexed == tokens
29
+
}
30
+
31
+
fn assert_errors(src: String, errors: List(pearl.Error)) -> Nil {
32
+
let #(_, found_errors) = src |> pearl.new |> pearl.tokenise
33
+
assert found_errors == errors
34
+
}
35
+
36
+
pub fn unknown_character_test() {
37
+
let src = "a&b"
38
+
assert_errors(src, [pearl.UnknownCharacter("&")])
39
+
}
40
+
41
+
pub fn unterminated_string_eof_test() {
42
+
let src = "\"Some string that doesn't end"
43
+
assert_errors(src, [pearl.UnterminatedString])
44
+
}
45
+
46
+
pub fn unterminated_string_newline_test() {
47
+
let src =
48
+
"\"Some string that tries
49
+
to continue on the next line"
50
+
assert_errors(src, [pearl.UnterminatedString])
51
+
}
52
+
53
+
pub fn unterminated_atom_test() {
54
+
let src = "'This is an atom!"
55
+
assert_errors(src, [pearl.UnterminatedAtom])
56
+
}
57
+
58
+
pub fn invalid_radix_test() {
59
+
let src = "94#123"
60
+
assert_errors(src, [pearl.InvalidRadix("94")])
61
+
}
62
+
63
+
pub fn invalid_radix_low_test() {
64
+
let src = "1#123"
65
+
assert_errors(src, [pearl.InvalidRadix("1")])
66
+
}
67
+
68
+
pub fn number_cannot_end_after_radix_test() {
69
+
let src = "16#"
70
+
assert_errors(src, [pearl.NumberCannotEndAfterRadix])
71
+
}
72
+
73
+
pub fn unterminated_character_test() {
74
+
let src = "$"
75
+
assert_errors(src, [pearl.UnterminatedCharacter])
76
+
}
77
+
78
+
pub fn expected_sigil_delimiter_test() {
79
+
let src = "~abc"
80
+
assert_errors(src, [pearl.ExpectedSigilDelimiter])
81
+
}
82
+
83
+
pub fn unterminated_escape_sequence_test() {
84
+
let src = "\"\\x\""
85
+
assert_errors(src, [pearl.UnterminatedEscapeSequence])
86
+
}
87
+
88
+
pub fn unterminated_escape_sequence2_test() {
89
+
let src = "\"\\x1\""
90
+
assert_errors(src, [pearl.UnterminatedEscapeSequence])
91
+
}
92
+
93
+
pub fn unterminated_escape_sequence3_test() {
94
+
let src = "\"\\x{123\""
95
+
assert_errors(src, [
96
+
pearl.UnterminatedEscapeSequence,
97
+
pearl.UnterminatedString,
98
+
])
99
+
}
100
+
101
+
pub fn unterminated_escape_sequence4_test() {
102
+
let src = "\"\\"
103
+
assert_errors(src, [
104
+
pearl.UnterminatedEscapeSequence,
105
+
pearl.UnterminatedString,
106
+
])
107
+
}
108
+
109
+
pub fn double_numeric_separator_test() {
110
+
let src = "1__2"
111
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
112
+
}
113
+
114
+
pub fn trailing_numeric_separator_test() {
115
+
let src = "1_2_"
116
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
117
+
}
118
+
119
+
pub fn trailing_decimal_numeric_separator_test() {
120
+
let src = "1_2_.3"
121
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
122
+
}
123
+
124
+
pub fn leading_decimal_numeric_separator_test() {
125
+
let src = "1_2._3"
126
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
127
+
}
128
+
129
+
pub fn trailing_exponent_numeric_separator_test() {
130
+
let src = "1_2.3_e4"
131
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
132
+
}
133
+
134
+
pub fn leading_exponent_numeric_separator_test() {
135
+
let src = "1_2.3e_4"
136
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
137
+
}
138
+
139
+
pub fn leading_negative_exponent_numeric_separator_test() {
140
+
let src = "1_2.3e-_4"
141
+
assert_errors(src, [pearl.NumericSeparatorNotAllowed])
142
+
}
143
+
144
+
pub fn missing_exponent_test() {
145
+
let src = "1.2e"
146
+
assert_errors(src, [pearl.ExpectedExponent])
147
+
}
148
+
149
+
pub fn missing_negative_exponent_test() {
150
+
let src = "1.2e-"
151
+
assert_errors(src, [pearl.ExpectedExponent])
12
152
}