mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Merge pull request #4582 from youknowone/unicode_names2

Browse Source 
Another temporary fix for #4566
This commit is contained in:
Jeong YunWon
2023-02-28 22:33:23 +09:00
committed by GitHub
parent 733ff9a05b c0f5266bc4
commit aa8336ee94
14 changed files with 353 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@@ -2933,9 +2933,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "unicode_names2"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301"
version = "0.6.0"
source = "git+https://github.com/youknowone/unicode_names2.git?tag=v0.6.0+character-alias#4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde"
dependencies = [
"phf",
]
[[package]]
name = "utf8parse"

1
Cargo.toml
View File

@@ -55,6 +55,7 @@ static_assertions = "1.1"
syn = "1.0.91"
thiserror = "1.0"
thread_local = "1.1.4"
unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", tag = "v0.6.0+character-alias" }
widestring = "0.5.1"
[features]

2
compiler/parser/Cargo.toml
View File

@@ -27,10 +27,10 @@ log = { workspace = true }
num-bigint = { workspace = true }
num-traits = { workspace = true }
thiserror = { workspace = true }
unicode_names2 = { workspace = true }
unic-emoji-char = "0.9.0"
unic-ucd-ident = "0.9.0"
unicode_names2 = "0.5.0"
lalrpop-util = "0.19.8"
phf = "0.11.1"
rustc-hash = "1.1.0"

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 15,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 15,
},
),
custom: (),
node: Constant {
value: Str(
"\u{8}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 9,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 9,
},
),
custom: (),
node: Constant {
value: Str(
"\u{7}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 21,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 21,
},
),
custom: (),
node: Constant {
value: Str(
"\r",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 45,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 45,
},
),
custom: (),
node: Constant {
value: Str(
"\u{89}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 12,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 12,
},
),
custom: (),
node: Constant {
value: Str(
"\u{7f}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 12,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 12,
},
),
custom: (),
node: Constant {
value: Str(
"\u{1b}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 15,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 15,
},
),
custom: (),
node: Constant {
value: Str(
"\u{c}",
),
kind: None,
},
},
},
},
]

40
compiler/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap generated Normal file
View File

@@ -0,0 +1,40 @@
---
source: compiler/parser/src/string.rs
expression: parse_ast
---
[
Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 9,
},
),
custom: (),
node: Expr {
value: Located {
location: Location {
row: 1,
column: 0,
},
end_location: Some(
Location {
row: 1,
column: 9,
},
),
custom: (),
node: Constant {
value: Str(
"\u{88}",
),
kind: None,
},
},
},
},
]

24
compiler/parser/src/string.rs
View File

@@ -1048,4 +1048,28 @@ mod tests {
let parse_ast = parse_program(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
macro_rules! test_aliases_parse {
($($name:ident: $alias:expr,)*) => {
$(
#[test]
fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = parse_program(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
)*
}
}
test_aliases_parse! {
test_backspace_alias: "BACKSPACE",
test_bell_alias: "BEL",
test_carriage_return_alias: "CARRIAGE RETURN",
test_delete_alias: "DELETE",
test_escape_alias: "ESCAPE",
test_form_feed_alias: "FORM FEED",
test_hts_alias: "HTS",
test_character_tabulation_with_justification_alias: "CHARACTER TABULATION WITH JUSTIFICATION",
}
}

2
stdlib/Cargo.toml
View File

@@ -57,7 +57,7 @@ sha3 = "0.10.1"
blake2 = "0.10.4"
## unicode stuff
unicode_names2 = "0.5.0"
unicode_names2 = { workspace = true }
# TODO: use unic for this; needed for title case:
# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
unicode-casing = "0.1.0"

2
vm/Cargo.toml
View File

@@ -81,7 +81,7 @@ sre-engine = "0.4.1"
# sre-engine = { path = "../../sre-engine" }
## unicode stuff
unicode_names2 = "0.5.1"
unicode_names2 = { workspace = true }
# TODO: use unic for this; needed for title case:
# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
unicode-casing = "0.1.0"