mirror of
https://github.com/RustPython/RustPython.git
synced 2026-06-02 19:39:49 +09:00
Compare commits
391 Commits
copilot/ad
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e35cc9e72 | ||
|
|
5b2f6bc270 | ||
|
|
b5b82587dd | ||
|
|
ef74577f4b | ||
|
|
14cb5dd874 | ||
|
|
a781980d9b | ||
|
|
6a860b635a | ||
|
|
0c9ed36b79 | ||
|
|
2c46b69e4b | ||
|
|
74b4707b65 | ||
|
|
885cf5c29c | ||
|
|
3e1c3bc86d | ||
|
|
36a1722d1b | ||
|
|
1ce37bf2d4 | ||
|
|
0e66439212 | ||
|
|
8018ba689f | ||
|
|
26f817a55b | ||
|
|
938d42184f | ||
|
|
1385c4e472 | ||
|
|
fc637a155d | ||
|
|
7d54ba502e | ||
|
|
e80a14ba12 | ||
|
|
30ae48b24b | ||
|
|
1a959cf7f3 | ||
|
|
ca412fce5d | ||
|
|
f95b7468f7 | ||
|
|
1cb24c5ebb | ||
|
|
ce79cd4853 | ||
|
|
dcb273ba68 | ||
|
|
dd8d250ba3 | ||
|
|
3db172c025 | ||
|
|
fb531ecce3 | ||
|
|
9701c46d86 | ||
|
|
021c78655e | ||
|
|
411230a46c | ||
|
|
b69644196a | ||
|
|
a6fee92683 | ||
|
|
f4696ea890 | ||
|
|
0237a1d520 | ||
|
|
6bbe24a725 | ||
|
|
f9e4988cf5 | ||
|
|
a5775e0c07 | ||
|
|
bc3d00e879 | ||
|
|
52305c0c72 | ||
|
|
7011942e4e | ||
|
|
438925401f | ||
|
|
2fabf38d8f | ||
|
|
d7d936575c | ||
|
|
b5ff41c219 | ||
|
|
e1d9a1123e | ||
|
|
b9efe10537 | ||
|
|
d3272e752b | ||
|
|
f3b83efcee | ||
|
|
1a013930a7 | ||
|
|
c513b923df | ||
|
|
cf3b6397b2 | ||
|
|
2a163609cd | ||
|
|
4eb9534646 | ||
|
|
ab5bc43359 | ||
|
|
06f73f2ae1 | ||
|
|
c845861c4f | ||
|
|
a136f9047b | ||
|
|
948924a14b | ||
|
|
ae6c16093e | ||
|
|
a6f13b98fb | ||
|
|
4daa526dc1 | ||
|
|
d073964aaa | ||
|
|
ee006af13e | ||
|
|
de06fc0923 | ||
|
|
ae3804fb21 | ||
|
|
d51069bc7f | ||
|
|
0063a6d18b | ||
|
|
cea21f953d | ||
|
|
62b081b893 | ||
|
|
26f5bbf077 | ||
|
|
ccf1508a06 | ||
|
|
c5143aa82f | ||
|
|
0a2461a704 | ||
|
|
276a0e6c42 | ||
|
|
d8dee81157 | ||
|
|
e8d7437d91 | ||
|
|
53941295a2 | ||
|
|
1fe4485c10 | ||
|
|
50f235aded | ||
|
|
b902ffdcf8 | ||
|
|
9c0557820b | ||
|
|
150b9103a8 | ||
|
|
48ad238349 | ||
|
|
67e66bdc75 | ||
|
|
e16f1aa657 | ||
|
|
6e56d935cf | ||
|
|
20cb8843e0 | ||
|
|
a4579a98b2 | ||
|
|
883ce9d273 | ||
|
|
723766ef69 | ||
|
|
3ed8c91fef | ||
|
|
a1a87dc8ca | ||
|
|
ea2a3d9d84 | ||
|
|
9c188b4da9 | ||
|
|
ab72b292bd | ||
|
|
f4f035013d | ||
|
|
c77c56d99c | ||
|
|
b432d4cbdc | ||
|
|
5a5230a400 | ||
|
|
8c988711dd | ||
|
|
f2ef252724 | ||
|
|
3a56f37505 | ||
|
|
4b9416a558 | ||
|
|
2d8f8ab818 | ||
|
|
a364f86fd5 | ||
|
|
7273d76cf2 | ||
|
|
54589bf255 | ||
|
|
0871bc8a2d | ||
|
|
f26016049d | ||
|
|
451bdcc9da | ||
|
|
8253253fc7 | ||
|
|
56269f704a | ||
|
|
078e23de27 | ||
|
|
ddfcb25957 | ||
|
|
e467b67ef7 | ||
|
|
460b1d39ed | ||
|
|
11e991fb95 | ||
|
|
ef375bec26 | ||
|
|
4059a032a7 | ||
|
|
e8711edd2d | ||
|
|
f8e0eeb579 | ||
|
|
32e6f8dd81 | ||
|
|
e36cd994e8 | ||
|
|
7ebffd063b | ||
|
|
5ef91c22de | ||
|
|
79395de9f3 | ||
|
|
2b19ba79a8 | ||
|
|
9d77b25858 | ||
|
|
d09179a7ee | ||
|
|
8fe23b8a15 | ||
|
|
ff05dae11c | ||
|
|
1da29ff003 | ||
|
|
a702e9ccc7 | ||
|
|
9ee27526bc | ||
|
|
fb37b5ecc6 | ||
|
|
3bfa5ab8c0 | ||
|
|
a0632ae2c5 | ||
|
|
f3c2198ff0 | ||
|
|
2bcbe96e6d | ||
|
|
f5357692e8 | ||
|
|
e0689bad7c | ||
|
|
1ac55db966 | ||
|
|
a7c9b98b5a | ||
|
|
2bb7dd8cf9 | ||
|
|
f6c382c595 | ||
|
|
72679af4b2 | ||
|
|
27db8e5847 | ||
|
|
68be554684 | ||
|
|
2e5077fe12 | ||
|
|
673db1d71a | ||
|
|
3cfb0fe7bc | ||
|
|
9e066c4f50 | ||
|
|
3bce5566fa | ||
|
|
3b67d0f5d2 | ||
|
|
77070eb6ca | ||
|
|
d6abdc4b79 | ||
|
|
fc00fc22d2 | ||
|
|
865e75dd99 | ||
|
|
9f019a5b86 | ||
|
|
ad6cc8f0a2 | ||
|
|
4a46e84eb9 | ||
|
|
949a620811 | ||
|
|
315865a6f7 | ||
|
|
c79aefecee | ||
|
|
3fbfbf53c2 | ||
|
|
3bd79e6b5b | ||
|
|
8e8b70fb33 | ||
|
|
29530049fe | ||
|
|
5b0177d20a | ||
|
|
479b2dc997 | ||
|
|
cb0cffbd7c | ||
|
|
cc829b2756 | ||
|
|
dfe99f647d | ||
|
|
b3098c3058 | ||
|
|
8f19dff19d | ||
|
|
320355f633 | ||
|
|
108461f637 | ||
|
|
bf7bb1bf3b | ||
|
|
09c27bb440 | ||
|
|
6dfc68b225 | ||
|
|
17376ace28 | ||
|
|
9eacdfca48 | ||
|
|
68a0bc030b | ||
|
|
edcf3002de | ||
|
|
67630ffbfe | ||
|
|
6ed17c3cb5 | ||
|
|
002fc1122e | ||
|
|
ae5c9119c9 | ||
|
|
4aa73ddab2 | ||
|
|
7576d68060 | ||
|
|
3bab7a9086 | ||
|
|
e10a27b1ae | ||
|
|
3b364608d9 | ||
|
|
1c4361803d | ||
|
|
22d4f43ad4 | ||
|
|
02a2b19839 | ||
|
|
ae7ff9c481 | ||
|
|
d877c30417 | ||
|
|
8bd4c5137e | ||
|
|
dc12bff0f4 | ||
|
|
e1dd3d43d5 | ||
|
|
a5cebc3242 | ||
|
|
ad5a55c1e3 | ||
|
|
e4d35b08ea | ||
|
|
02932384d6 | ||
|
|
0325fd429e | ||
|
|
4db0aca471 | ||
|
|
83002d7369 | ||
|
|
90cc888f4b | ||
|
|
cf23884656 | ||
|
|
c3c9292c8b | ||
|
|
ee5e9d0001 | ||
|
|
181e4e7124 | ||
|
|
eb99a8ecf1 | ||
|
|
acc167fc44 | ||
|
|
c2910c06f3 | ||
|
|
ac3e067230 | ||
|
|
be7841f9c1 | ||
|
|
3e66fb508a | ||
|
|
f2ad84a489 | ||
|
|
926d69b50a | ||
|
|
8d1c68c9a0 | ||
|
|
d9fff99718 | ||
|
|
0f25d145fd | ||
|
|
7fd0da92d3 | ||
|
|
c027ffc2ba | ||
|
|
192ba371e4 | ||
|
|
d25195ed0e | ||
|
|
9004089fd1 | ||
|
|
f6e2fcffe7 | ||
|
|
bb77ac6284 | ||
|
|
c2141a765f | ||
|
|
dd1cbac692 | ||
|
|
c98d26e508 | ||
|
|
5e408d65f4 | ||
|
|
73b5b69bd8 | ||
|
|
e79df4a6fb | ||
|
|
32e16fe7da | ||
|
|
88be7bb16a | ||
|
|
f90a5cf650 | ||
|
|
6c2c8421d7 | ||
|
|
6e895fbac4 | ||
|
|
543fcc841c | ||
|
|
6c91c5bb2a | ||
|
|
3c297d478a | ||
|
|
6ed2f15b67 | ||
|
|
6b67067e9a | ||
|
|
04ffa3891c | ||
|
|
ba2b619c0c | ||
|
|
c8ddbd2326 | ||
|
|
3ebcab70c0 | ||
|
|
51e7200d11 | ||
|
|
d7a319d967 | ||
|
|
330b18f2fe | ||
|
|
82e8b200db | ||
|
|
3f718f9942 | ||
|
|
363d19839f | ||
|
|
68aece59c9 | ||
|
|
b3d6d2f247 | ||
|
|
e6d9ea6bfe | ||
|
|
59382f385a | ||
|
|
9db00f741c | ||
|
|
e5f2d2d3b9 | ||
|
|
7fb743b1be | ||
|
|
6c498fc4a7 | ||
|
|
b8f7ae4265 | ||
|
|
1d42ee565f | ||
|
|
9794ab7fdf | ||
|
|
dc81c740cf | ||
|
|
f10f441854 | ||
|
|
1fa676fd07 | ||
|
|
5648a3346f | ||
|
|
02c454bdb4 | ||
|
|
049d44b1e0 | ||
|
|
f6b6b18b62 | ||
|
|
7f8cdddbbf | ||
|
|
625e5bf012 | ||
|
|
a2afaf0f13 | ||
|
|
956267c49e | ||
|
|
be43bb6dbf | ||
|
|
6ab1f806ba | ||
|
|
f0e23aacc2 | ||
|
|
a5f48eaaa1 | ||
|
|
b427f31164 | ||
|
|
7df0801db3 | ||
|
|
3a793ce716 | ||
|
|
adafaf222b | ||
|
|
f6371de4a1 | ||
|
|
fb1218d6ba | ||
|
|
3f8a0b12eb | ||
|
|
2e5c2be7fa | ||
|
|
0d67fd69e2 | ||
|
|
952be48944 | ||
|
|
0913563bbe | ||
|
|
1ab76d012b | ||
|
|
f0acc67855 | ||
|
|
9ebdf10c11 | ||
|
|
7bb2fb0755 | ||
|
|
43ef2eabbe | ||
|
|
dc0c814671 | ||
|
|
a7ea01a135 | ||
|
|
cbfb313de2 | ||
|
|
adb169e65b | ||
|
|
5081f76faf | ||
|
|
f2e055f7d6 | ||
|
|
f2f20175b3 | ||
|
|
a693a0c8aa | ||
|
|
71380bead9 | ||
|
|
5a45d41df0 | ||
|
|
9b0c668f74 | ||
|
|
dc65255fd2 | ||
|
|
a9fd4bf41f | ||
|
|
5058090a3f | ||
|
|
b929a50647 | ||
|
|
f8862e4eed | ||
|
|
18c6c16e2a | ||
|
|
d5921d16af | ||
|
|
9140ef583a | ||
|
|
af41d11faf | ||
|
|
175f12b664 | ||
|
|
b18b71b2db | ||
|
|
fdb49d83c5 | ||
|
|
37707081f8 | ||
|
|
764e4de061 | ||
|
|
b842a6c6c6 | ||
|
|
9669118d3c | ||
|
|
67eedddcd7 | ||
|
|
57ca1d59a6 | ||
|
|
9a0410dab4 | ||
|
|
b80c2bd5ec | ||
|
|
a5b9f0e80b | ||
|
|
caf8d55da5 | ||
|
|
c79baa3317 | ||
|
|
f0bf8100c9 | ||
|
|
1f1be5e29e | ||
|
|
4f1cf6d401 | ||
|
|
3e1aa7cbe6 | ||
|
|
b9f9ba145e | ||
|
|
3d91197b38 | ||
|
|
2827eca293 | ||
|
|
aac207003f | ||
|
|
f82b8d8eb7 | ||
|
|
8d61a2217b | ||
|
|
640cbd7c4a | ||
|
|
aa12accdac | ||
|
|
fd2117355e | ||
|
|
36025386f3 | ||
|
|
330aa08488 | ||
|
|
a2c3e65b81 | ||
|
|
8108b6a153 | ||
|
|
63a1c0e95c | ||
|
|
c98939a7c1 | ||
|
|
9f1429d95f | ||
|
|
73218f42d5 | ||
|
|
f197699e3c | ||
|
|
898da7f58c | ||
|
|
0ee07e3d0a | ||
|
|
49048504f6 | ||
|
|
e3997ad1b8 | ||
|
|
da01e617de | ||
|
|
891538d924 | ||
|
|
3096d77ec5 | ||
|
|
9e2d03428c | ||
|
|
d1e9763ff3 | ||
|
|
2b1b0ba805 | ||
|
|
e09b66dd86 | ||
|
|
a24ee58961 | ||
|
|
1721f62804 | ||
|
|
8f71ff4b47 | ||
|
|
4bbabe5810 | ||
|
|
7544628268 | ||
|
|
7e637e8cbd | ||
|
|
7e5e026941 | ||
|
|
27aed85599 | ||
|
|
d201c48e1c | ||
|
|
9cf7bcd64a | ||
|
|
31edcfa97e | ||
|
|
eac45727d2 | ||
|
|
28acbc66f9 | ||
|
|
a49ce5bf34 | ||
|
|
7b5ac61026 | ||
|
|
ad66d9acd0 | ||
|
|
00dd9a5ed1 | ||
|
|
d5a90e5c1f | ||
|
|
72f397c6df | ||
|
|
e009cc0c3b |
60
.agents/skills/rustpython-capi-expansion/SKILL.md
Normal file
60
.agents/skills/rustpython-capi-expansion/SKILL.md
Normal file
@@ -0,0 +1,60 @@
|
||||
---
|
||||
name: rustpython-capi-expansion
|
||||
description: Implement missing RustPython C-API functions in crates/capi using the pyo3-ffi header split mapping (`pyo3-ffi/src/*.rs`, mirroring CPython C API headers). Use this whenever the user asks to add or port C-API functions (for example from setobject.h, dictobject.h, unicodeobject.h) or add capi tests.
|
||||
---
|
||||
|
||||
# RustPython C-API Expansion
|
||||
|
||||
Use this workflow for adding missing C-API functions to RustPython.
|
||||
|
||||
## Source of truth for target files
|
||||
|
||||
- Use this mapping source: `pyo3-ffi/src/*.rs`, which mirrors the CPython header split used by the C API.
|
||||
- Map requested header APIs to `crates/capi/src/<header_basename>.rs` using that split. Examples:
|
||||
- `setobject.h` -> `crates/capi/src/setobject.rs`
|
||||
- `dictobject.h` -> `crates/capi/src/dictobject.rs`
|
||||
- `unicodeobject.h` -> `crates/capi/src/unicodeobject.rs`
|
||||
- Do not invent alternate target modules when the header split implies a direct target.
|
||||
- If the target file is not present yet, create it and wire it in `crates/capi/src/lib.rs`.
|
||||
|
||||
## Implementation workflow
|
||||
|
||||
1. Identify requested missing APIs from the user request and their originating C API header.
|
||||
2. Open nearby capi modules in `crates/capi/src/` and follow existing style and patterns.
|
||||
3. Implement only the requested functions in the mapped target file.
|
||||
4. Keep behavior aligned with CPython C-API contracts.
|
||||
5. Prefer using existing `rustpython-vm` functionality as much as possible instead of re-implementing behavior in capi.
|
||||
6. If a needed `rustpython-vm` helper exists but is private, make it public with a minimal, focused visibility change.
|
||||
7. Prefer direct contract assumptions over defensive null checks unless required by the established local style.
|
||||
8. Add basic tests only; do not overfit with very specific edge-case clutter.
|
||||
9. In tests, use `pyo3` only as a safe wrapper over the API. Avoid raw pointer-heavy direct FFI-style tests.
|
||||
10. Run tests from `crates/capi`.
|
||||
|
||||
## Testing rules
|
||||
|
||||
- Run test commands with working directory set to `crates/capi`.
|
||||
- Prefer targeted tests first (module/function filter), then broader capi tests if needed.
|
||||
- Keep test names concise (no required `test_` prefix).
|
||||
|
||||
## Style rules
|
||||
|
||||
- Follow existing RustPython capi coding style in neighboring files.
|
||||
- Reuse `rustpython-vm` methods and types first; avoid duplicating VM logic in capi wrappers.
|
||||
- When exposing previously private VM helpers, keep the API surface minimal and avoid unrelated refactors.
|
||||
- Only expose and implement ABI-stable C-API surface needed for `abi3` / `abi3t`.
|
||||
- Add comments only when they explain non-obvious behavior.
|
||||
- Keep edits minimal and focused on requested API expansion.
|
||||
|
||||
## Completion checklist
|
||||
|
||||
- [ ] All requested functions implemented in mapped target file.
|
||||
- [ ] New module exported in `crates/capi/src/lib.rs` when applicable.
|
||||
- [ ] Basic safe-wrapper `pyo3` tests added/updated.
|
||||
- [ ] Tests executed from `crates/capi` and passing for changed area.
|
||||
- [ ] Final response includes changed file paths and test command summary.
|
||||
|
||||
## Example prompts this skill should handle
|
||||
|
||||
- "Implement these missing functions from `dictobject.h`."
|
||||
- "Add `setobject.h` C-API functions in RustPython and include basic tests."
|
||||
- "Port the listed `unicodeobject.h` APIs in capi, follow existing style, and run tests from `crates/capi`."
|
||||
@@ -4,6 +4,8 @@ argdefs
|
||||
argtypes
|
||||
asdl
|
||||
asname
|
||||
atopen
|
||||
atext
|
||||
attro
|
||||
augassign
|
||||
badcert
|
||||
@@ -30,9 +32,12 @@ cellvar
|
||||
cellvars
|
||||
ceval
|
||||
cfield
|
||||
cfws
|
||||
CFWS
|
||||
CLASSDEREF
|
||||
classdict
|
||||
cmpop
|
||||
CNOTAB
|
||||
codedepth
|
||||
CODEUNIT
|
||||
CONIN
|
||||
@@ -47,13 +52,16 @@ datastack
|
||||
defaultdict
|
||||
denom
|
||||
deopt
|
||||
deopts
|
||||
dictbytype
|
||||
DICTFLAG
|
||||
dictoffset
|
||||
distpoint
|
||||
dynload
|
||||
elts
|
||||
eooh
|
||||
eofs
|
||||
EOOH
|
||||
evalloop
|
||||
excepthandler
|
||||
exceptiontable
|
||||
@@ -62,6 +70,7 @@ fastlocals
|
||||
fblock
|
||||
fblocks
|
||||
fdescr
|
||||
fdst
|
||||
ffi_argtypes
|
||||
fielddesc
|
||||
fieldlist
|
||||
@@ -75,6 +84,7 @@ freelist
|
||||
freevar
|
||||
freevars
|
||||
fromlist
|
||||
fsrc
|
||||
getdict
|
||||
getfunc
|
||||
getiter
|
||||
@@ -89,28 +99,39 @@ HASUNION
|
||||
heaptype
|
||||
hexdigit
|
||||
HIGHRES
|
||||
ialloc
|
||||
IFUNC
|
||||
IMMUTABLETYPE
|
||||
INCREF
|
||||
inlinedepth
|
||||
inplace
|
||||
inpos
|
||||
ioffset
|
||||
isbytecode
|
||||
ishidden
|
||||
ismine
|
||||
ISPOINTER
|
||||
isoctal
|
||||
iteminfo
|
||||
Itertool
|
||||
iused
|
||||
keeped
|
||||
kwnames
|
||||
kwonlyarg
|
||||
kwonlyargs
|
||||
kwonlydefaults
|
||||
lasti
|
||||
libffi
|
||||
linearise
|
||||
lineful
|
||||
lineiterator
|
||||
linetable
|
||||
LNOTAB
|
||||
loadfast
|
||||
localsplus
|
||||
localspluskinds
|
||||
Lshift
|
||||
lslpp
|
||||
lsprof
|
||||
MAXBLOCKS
|
||||
maxdepth
|
||||
@@ -120,16 +141,23 @@ mult
|
||||
multibytecodec
|
||||
nameobj
|
||||
nameop
|
||||
nargsf
|
||||
nblocks
|
||||
ncells
|
||||
ncellsused
|
||||
ncellvars
|
||||
nconsts
|
||||
newargs
|
||||
newfree
|
||||
NEWLOCALS
|
||||
newsemlockobject
|
||||
nextop
|
||||
nfrees
|
||||
nkwargs
|
||||
nkwelts
|
||||
nlocalsplus
|
||||
nointerrupt
|
||||
noffsets
|
||||
Nondescriptor
|
||||
noninteger
|
||||
nops
|
||||
@@ -137,6 +165,7 @@ noraise
|
||||
nseen
|
||||
NSIGNALS
|
||||
numer
|
||||
nvars
|
||||
opname
|
||||
opnames
|
||||
orelse
|
||||
@@ -149,18 +178,22 @@ patma
|
||||
peepholer
|
||||
phcount
|
||||
platstdlib
|
||||
ploc
|
||||
posonlyarg
|
||||
posonlyargs
|
||||
prec
|
||||
preds
|
||||
preinitialized
|
||||
pybuilddir
|
||||
pycore
|
||||
pyinner
|
||||
pydecimal
|
||||
pyerrors
|
||||
Pyfunc
|
||||
pylifecycle
|
||||
pymain
|
||||
pyrepl
|
||||
pystate
|
||||
PYTHONTRACEMALLOC
|
||||
PYTHONUTF8
|
||||
pythonw
|
||||
@@ -168,6 +201,7 @@ PYTHREAD_NAME
|
||||
releasebuffer
|
||||
repr
|
||||
resinfo
|
||||
retarget
|
||||
Rshift
|
||||
SA_ONSTACK
|
||||
saveall
|
||||
@@ -179,6 +213,7 @@ SETREF
|
||||
setresult
|
||||
setslice
|
||||
settraceallthreads
|
||||
sget
|
||||
SLOTDEFINED
|
||||
SMALLBUF
|
||||
SOABI
|
||||
@@ -189,13 +224,16 @@ staticbase
|
||||
stginfo
|
||||
storefast
|
||||
stringlib
|
||||
stringized
|
||||
structseq
|
||||
subkwargs
|
||||
subparams
|
||||
subscr
|
||||
sval
|
||||
swappedbytes
|
||||
swaptimize
|
||||
sysdict
|
||||
tbstderr
|
||||
templatelib
|
||||
testconsole
|
||||
threadstate
|
||||
@@ -214,6 +252,7 @@ uncollectable
|
||||
Unhandle
|
||||
unparse
|
||||
unparser
|
||||
untargeted
|
||||
untracking
|
||||
VARKEYWORDS
|
||||
varkwarg
|
||||
|
||||
@@ -67,6 +67,7 @@ fillchar
|
||||
fillvalue
|
||||
finallyhandler
|
||||
firstiter
|
||||
fobj
|
||||
firstlineno
|
||||
fnctl
|
||||
frombytes
|
||||
@@ -111,12 +112,14 @@ idfunc
|
||||
idiv
|
||||
idxs
|
||||
impls
|
||||
infd
|
||||
indexgroup
|
||||
infj
|
||||
inittab
|
||||
Inittab
|
||||
instancecheck
|
||||
instanceof
|
||||
instrs
|
||||
interpchannels
|
||||
interpqueues
|
||||
irepeat
|
||||
@@ -175,6 +178,7 @@ Nonprintable
|
||||
onceregistry
|
||||
origname
|
||||
ospath
|
||||
outfd
|
||||
pendingcr
|
||||
phello
|
||||
platlibdir
|
||||
@@ -185,10 +189,12 @@ posonlyargcount
|
||||
prepending
|
||||
profilefunc
|
||||
pycache
|
||||
pycapsule
|
||||
pycodecs
|
||||
pycs
|
||||
pydatetime
|
||||
pyexpat
|
||||
PYGILSTATE
|
||||
pyio
|
||||
pymain
|
||||
PYTHONAPI
|
||||
|
||||
12
.cspell.json
12
.cspell.json
@@ -49,20 +49,25 @@
|
||||
"ignorePaths": [
|
||||
"**/__pycache__/**",
|
||||
"target/**",
|
||||
"Lib/**"
|
||||
"Lib/**",
|
||||
"crates/host_env/**"
|
||||
],
|
||||
// words - list of words to be always considered correct
|
||||
// (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords)
|
||||
"words": [
|
||||
"aiterable",
|
||||
"alnum",
|
||||
"csock",
|
||||
"coro",
|
||||
"dedentations",
|
||||
"dedents",
|
||||
"deduped",
|
||||
"deoptimized",
|
||||
"deoptimize",
|
||||
"emscripten",
|
||||
"excs",
|
||||
"fnfe",
|
||||
"ifexp",
|
||||
"interps",
|
||||
"jitted",
|
||||
"jitting",
|
||||
@@ -72,8 +77,13 @@
|
||||
"oparg",
|
||||
"opargs",
|
||||
"pyc",
|
||||
"reborrow",
|
||||
"reraises",
|
||||
"reraising",
|
||||
"significand",
|
||||
"summands",
|
||||
"TESTFN",
|
||||
"TZPATH",
|
||||
"unraisable",
|
||||
"wasi",
|
||||
"weaked",
|
||||
|
||||
19
.gitattributes
vendored
19
.gitattributes
vendored
@@ -58,13 +58,14 @@ Lib/venv/scripts/posix/* text eol=lf
|
||||
#
|
||||
[attr]generated linguist-generated=true diff=generated
|
||||
|
||||
Lib/_opcode_metadata.py generated
|
||||
Lib/keyword.py generated
|
||||
Lib/idlelib/help.html generated
|
||||
Lib/test/certdata/*.pem generated
|
||||
Lib/test/certdata/*.0 generated
|
||||
Lib/test/levenshtein_examples.json generated
|
||||
Lib/test/test_stable_abi_ctypes.py generated
|
||||
Lib/token.py generated
|
||||
Lib/_opcode_metadata.py generated
|
||||
Lib/keyword.py generated
|
||||
Lib/idlelib/help.html generated
|
||||
Lib/test/certdata/*.pem generated
|
||||
Lib/test/certdata/*.0 generated
|
||||
Lib/test/levenshtein_examples.json generated
|
||||
Lib/test/test_stable_abi_ctypes.py generated
|
||||
Lib/token.py generated
|
||||
crates/compiler-core/src/bytecode/opcode_metadata.rs generated
|
||||
|
||||
.github/workflows/*.lock.yml linguist-generated=true merge=ours
|
||||
.github/workflows/*.lock.yml linguist-generated=true merge=ours
|
||||
|
||||
10
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
10
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
<!--
|
||||
Thanks for your contribution!
|
||||
-->
|
||||
|
||||
- [ ] Closes #xxxx <!-- Replace xxxx with the GitHub issue number -->
|
||||
- [ ] This PR follows our [AI policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md)
|
||||
|
||||
## Summary
|
||||
<!-- What's the purpose of the change? What does it do, and why? -->
|
||||
|
||||
10
.github/dependabot.yml
vendored
10
.github/dependabot.yml
vendored
@@ -2,7 +2,9 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: cargo
|
||||
directory: /
|
||||
directories:
|
||||
- "/"
|
||||
- "crates/*"
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
@@ -19,6 +21,7 @@ updates:
|
||||
- "digest"
|
||||
- "md-5"
|
||||
- "sha-1"
|
||||
- "sha1"
|
||||
- "sha2"
|
||||
- "sha3"
|
||||
- "blake2"
|
||||
@@ -120,6 +123,11 @@ updates:
|
||||
toml:
|
||||
patterns:
|
||||
- "toml*"
|
||||
unix:
|
||||
patterns:
|
||||
- "mac_address"
|
||||
- "nix"
|
||||
- "rustyline"
|
||||
wasm-bindgen:
|
||||
patterns:
|
||||
- "wasm-bindgen*"
|
||||
|
||||
450
.github/workflows/ci.yaml
vendored
450
.github/workflows/ci.yaml
vendored
@@ -19,21 +19,62 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env
|
||||
CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env
|
||||
CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env
|
||||
# Crates excluded from workspace builds:
|
||||
# - rustpython_wasm: requires wasm target
|
||||
# - rustpython-compiler-source: deprecated
|
||||
# - rustpython-venvlauncher: Windows-only
|
||||
WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher
|
||||
# Python version targeted by the CI.
|
||||
PYTHON_VERSION: "3.14.3"
|
||||
X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
|
||||
X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
|
||||
CARGO_INCREMENTAL: 0
|
||||
CARGO_PROFILE_TEST_DEBUG: 0
|
||||
CARGO_PROFILE_DEV_DEBUG: 0
|
||||
CARGO_PROFILE_RELEASE_DEBUG: 0
|
||||
CARGO_TERM_COLOR: always
|
||||
CI: true
|
||||
|
||||
jobs:
|
||||
determine_changes:
|
||||
name: Determine changes
|
||||
runs-on: ubuntu-slim
|
||||
outputs:
|
||||
# Flag that is raised when any rust code is changed.
|
||||
rust_code: ${{ steps.check_rust_code.outputs.changed }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Determine merge base
|
||||
id: merge_base
|
||||
run: |
|
||||
sha=$(git merge-base HEAD "origin/${BASE_REF}")
|
||||
echo "sha=${sha}" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
|
||||
|
||||
- name: Check if there was any code related change
|
||||
id: check_rust_code
|
||||
run: |
|
||||
if git diff --quiet "${MERGE_BASE}...HEAD" -- \
|
||||
':Cargo.toml' \
|
||||
':Cargo.lock' \
|
||||
':rust-toolchain.toml' \
|
||||
':.cargo/config.toml' \
|
||||
':crates/**' \
|
||||
':src/**' \
|
||||
':.github/workflows/ci.yaml' \
|
||||
; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
|
||||
|
||||
rust_tests:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
env:
|
||||
@@ -51,27 +92,35 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
# Windows runners randomly crashes, https://github.com/actions/cache/issues/1754
|
||||
continue-on-error: true
|
||||
|
||||
- name: Install macOS dependencies
|
||||
uses: ./.github/actions/install-macos-deps
|
||||
|
||||
- name: run clippy
|
||||
run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings
|
||||
|
||||
- name: run rust tests
|
||||
run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }}
|
||||
run: cargo test --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }}
|
||||
env:
|
||||
INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
|
||||
|
||||
- name: check compilation without threading
|
||||
run: cargo check ${{ env.CARGO_ARGS }}
|
||||
|
||||
- run: cargo doc --locked
|
||||
if: runner.os == 'Linux'
|
||||
- name: run c-api tests
|
||||
working-directory: crates/capi
|
||||
run: cargo test
|
||||
if: runner.os != 'Windows' # Requires pyo3 0.29+ on Windows
|
||||
|
||||
- name: check compilation without host_env (sandbox mode)
|
||||
run: |
|
||||
@@ -90,6 +139,10 @@ jobs:
|
||||
run: cargo build --no-default-features --features ssl-openssl
|
||||
if: runner.os == 'Linux'
|
||||
|
||||
- name: Test vendored OpenSSL build
|
||||
run: cargo build --no-default-features --features ssl-openssl-vendor
|
||||
if: runner.os == 'Linux'
|
||||
|
||||
# - name: Install tk-dev for tkinter build
|
||||
# run: sudo apt-get update && sudo apt-get install -y tk-dev
|
||||
# if: runner.os == 'Linux'
|
||||
@@ -111,12 +164,18 @@ jobs:
|
||||
if: runner.os == 'Linux'
|
||||
|
||||
cargo_check:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: cargo check
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs:
|
||||
- determine_changes
|
||||
if: |
|
||||
(
|
||||
!contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
|
||||
needs.determine_changes.outputs.rust_code == 'true'
|
||||
) || github.ref == 'refs/heads/main'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-linux-android
|
||||
- os: ubuntu-latest
|
||||
@@ -127,10 +186,13 @@ jobs:
|
||||
target: i686-unknown-linux-musl
|
||||
dependencies:
|
||||
musl-tools: true
|
||||
skip_ssl: true
|
||||
- os: ubuntu-latest
|
||||
target: wasm32-wasip2
|
||||
skip_ssl: true
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-freebsd
|
||||
skip_ssl: true
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
dependencies:
|
||||
@@ -155,8 +217,7 @@ jobs:
|
||||
gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }}
|
||||
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
@@ -165,9 +226,12 @@ jobs:
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
# key won't match, will rely on restore-keys
|
||||
key: cargo-check-${{ runner.os }}-${{ matrix.target }}
|
||||
key: ${{ runner.os }}-${{ matrix.target }}
|
||||
restore-keys: |
|
||||
cargo-check-${{ runner.os }}-${{ matrix.target }}-
|
||||
${{ runner.os }}-stable-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable-${{ matrix.target }}-
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
@@ -176,11 +240,25 @@ jobs:
|
||||
- name: Setup Android NDK
|
||||
if: ${{ matrix.target == 'aarch64-linux-android' }}
|
||||
id: setup-ndk
|
||||
uses: nttld/setup-ndk@v1
|
||||
uses: nttld/setup-ndk@ed92fe6cadad69be94a966a7ee3271275e62f779 # v1.6.0
|
||||
with:
|
||||
ndk-version: r27
|
||||
add-to-path: true
|
||||
|
||||
- name: Append env conf to cargo
|
||||
if: ${{ matrix.target == 'aarch64-linux-android' }}
|
||||
env:
|
||||
NDK_PATH: ${{ steps.setup-ndk.outputs.ndk-path }}
|
||||
run: |
|
||||
{
|
||||
echo "[env]"
|
||||
echo "CC_aarch64_linux_android = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang\""
|
||||
|
||||
echo "AR_aarch64_linux_android = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar\""
|
||||
|
||||
echo "CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang\""
|
||||
} >> .cargo/config.toml
|
||||
|
||||
# - name: Prepare repository for redox compilation
|
||||
# run: bash scripts/redox/uncomment-cargo.sh
|
||||
# - name: Check compilation for Redox
|
||||
@@ -189,24 +267,12 @@ jobs:
|
||||
# command: check
|
||||
# args: --ignore-rust-version
|
||||
|
||||
- name: Check compilation
|
||||
run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }}
|
||||
env:
|
||||
CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang
|
||||
AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar
|
||||
CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang
|
||||
- name: Check compilation with threading
|
||||
run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} --features threading
|
||||
|
||||
- name: Save cache
|
||||
if: ${{ github.ref == 'refs/heads/main' }} # only save on main
|
||||
uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }}
|
||||
- name: Check compilation with ssl
|
||||
if: ${{ !matrix.skip_ssl }}
|
||||
run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS }}
|
||||
|
||||
snippets_cpython:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
@@ -228,23 +294,22 @@ jobs:
|
||||
- os: macos-latest
|
||||
extra_test_args:
|
||||
- '-u all'
|
||||
env_polluting_tests: []
|
||||
env_polluting_tests:
|
||||
- test_set
|
||||
skips: []
|
||||
timeout: 50
|
||||
- os: ubuntu-latest
|
||||
extra_test_args:
|
||||
- '-u all'
|
||||
env_polluting_tests: []
|
||||
env_polluting_tests:
|
||||
- test_set
|
||||
skips: []
|
||||
timeout: 60
|
||||
- os: windows-2025
|
||||
extra_test_args: [] # TODO: Enable '-u all'
|
||||
env_polluting_tests: []
|
||||
skips:
|
||||
- test_rlcompleter
|
||||
- test_pathlib # panic by surrogate chars
|
||||
- test_posixpath # OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)')
|
||||
- test_venv # couple of failing tests
|
||||
env_polluting_tests:
|
||||
- test_set
|
||||
skips: []
|
||||
timeout: 50
|
||||
fail-fast: false
|
||||
steps:
|
||||
@@ -254,13 +319,23 @@ jobs:
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
# Windows runners randomly crashes, https://github.com/actions/cache/issues/1754
|
||||
continue-on-error: true
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install macOS dependencies
|
||||
uses: ./.github/actions/install-macos-deps
|
||||
@@ -290,8 +365,24 @@ jobs:
|
||||
|
||||
- name: Run flaky MP CPython tests
|
||||
run: |
|
||||
target/release/rustpython -m test -j 1 ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v ${{ env.FLAKY_MP_TESTS }}
|
||||
for attempt in $(seq 1 5); do
|
||||
echo "::group::Attempt ${attempt}"
|
||||
|
||||
set +e
|
||||
target/release/rustpython -m test -j 1 ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v ${{ env.FLAKY_MP_TESTS }}
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
echo "::endgroup::"
|
||||
|
||||
if [ $status -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
exit 1
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
shell: bash
|
||||
env:
|
||||
RUSTPYTHON_SKIP_ENV_POLLUTERS: true
|
||||
|
||||
@@ -348,12 +439,82 @@ jobs:
|
||||
shell: bash
|
||||
run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit
|
||||
|
||||
clippy:
|
||||
name: clippy
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs:
|
||||
- determine_changes
|
||||
permissions:
|
||||
contents: read
|
||||
if: |
|
||||
needs.determine_changes.outputs.rust_code == 'true' ||
|
||||
github.ref == 'refs/heads/main'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os:
|
||||
- macos-latest
|
||||
- ubuntu-latest
|
||||
- windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
# Windows runners randomly crashes, https://github.com/actions/cache/issues/1754
|
||||
continue-on-error: true
|
||||
|
||||
- name: Clippy
|
||||
run: cargo clippy --keep-going ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings
|
||||
|
||||
cargo_shear:
|
||||
name: cargo shear
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- determine_changes
|
||||
permissions:
|
||||
contents: read
|
||||
if: |
|
||||
needs.determine_changes.outputs.rust_code == 'true' ||
|
||||
github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: cargo-bins/cargo-binstall@aaa84a43aec4955a42c5ffc65d258961e39f276e # v1.19.1
|
||||
|
||||
- name: cargo shear
|
||||
run: |
|
||||
cargo binstall --no-confirm cargo-shear
|
||||
cargo shear
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
security-events: write # for zizmor
|
||||
steps:
|
||||
@@ -362,54 +523,70 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- uses: cargo-bins/cargo-binstall@113a77a4ce971c41332f2129c3d995df993cf746 # v1.17.8
|
||||
|
||||
- name: cargo shear
|
||||
run: |
|
||||
cargo binstall --no-confirm cargo-shear
|
||||
cargo shear
|
||||
|
||||
- name: actionlint
|
||||
uses: reviewdog/action-actionlint@0d952c597ef8459f634d7145b0b044a9699e5e43 # v1.71.0
|
||||
uses: reviewdog/action-actionlint@6fb7acc99f4a1008869fa8a0f09cfca740837d9d # v1.72.0
|
||||
|
||||
- name: zizmor
|
||||
uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2
|
||||
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||
|
||||
- name: restore prek cache
|
||||
if: ${{ github.ref != 'refs/heads/main' }} # never restore on main
|
||||
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
|
||||
path: ~/.cache/prek
|
||||
|
||||
- name: prek
|
||||
- name: install prek
|
||||
id: prek
|
||||
uses: j178/prek-action@53276d8b0d10f8b6672aa85b4588c6921d0370cc # v2.0.1
|
||||
uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4
|
||||
with:
|
||||
cache: false
|
||||
show-verbose-logs: false
|
||||
continue-on-error: true
|
||||
install-only: true
|
||||
|
||||
- name: prek run
|
||||
run: prek run --show-diff-on-failure --color=always --all-files
|
||||
|
||||
- name: Get target CPython version
|
||||
id: cpython-version
|
||||
run: |
|
||||
version=$(cat .python-version)
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Clone CPython
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: python/cpython
|
||||
path: cpython
|
||||
ref: "v${{ steps.cpython-version.outputs.version }}"
|
||||
persist-credentials: false
|
||||
|
||||
- name: prek run (manual stage)
|
||||
run: prek run --show-diff-on-failure --color=always --all-files --hook-stage manual
|
||||
env:
|
||||
CPYTHON_ROOT: ${{ github.workspace }}/cpython
|
||||
|
||||
- name: save prek cache
|
||||
if: ${{ github.ref == 'refs/heads/main' }} # only save on main
|
||||
uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
|
||||
path: ~/.cache/prek
|
||||
|
||||
- name: restore git permissions
|
||||
if: ${{ !cancelled() }}
|
||||
run: sudo chown -R "$(id -u):$(id -g)" .git
|
||||
|
||||
- name: reviewdog
|
||||
uses: reviewdog/action-suggester@aa38384ceb608d00f84b4690cacc83a5aba307ff # 1.24.0
|
||||
if: ${{ !cancelled() }}
|
||||
uses: reviewdog/action-suggester@aa38384ceb608d00f84b4690cacc83a5aba307ff # v1.24.0
|
||||
with:
|
||||
level: warning
|
||||
fail_level: error
|
||||
cleanup: false
|
||||
|
||||
miri:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
@@ -428,9 +605,18 @@ jobs:
|
||||
toolchain: ${{ env.NIGHTLY_CHANNEL }}
|
||||
components: miri
|
||||
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-
|
||||
|
||||
- name: Run tests under miri
|
||||
run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
|
||||
@@ -453,12 +639,23 @@ jobs:
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
${{ runner.os }}-
|
||||
|
||||
- name: cargo clippy
|
||||
run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings
|
||||
run: cargo clippy --keep-going --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings
|
||||
|
||||
- name: install wasm-pack
|
||||
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
|
||||
@@ -469,15 +666,29 @@ jobs:
|
||||
tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- run: python -m pip install -r requirements.txt
|
||||
working-directory: ./wasm/tests
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
cache: "npm"
|
||||
cache-dependency-path: "wasm/demo/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Get npm cache directory
|
||||
id: npm-cache-dir
|
||||
shell: bash
|
||||
run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore npm cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
# don't restore on main or release
|
||||
if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release'
|
||||
with:
|
||||
path: ${{ steps.npm-cache-dir.outputs.dir }}
|
||||
key: node-${{ runner.os }}-wasm-demo-
|
||||
restore-keys: |
|
||||
node-${{ runner.os }}-wasm-demo-
|
||||
|
||||
- name: run test
|
||||
run: |
|
||||
driver_path="$(pwd)/../../geckodriver"
|
||||
@@ -487,8 +698,11 @@ jobs:
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/demo
|
||||
- uses: mwilliamson/setup-wabt-action@v3
|
||||
with: { wabt-version: "1.0.36" }
|
||||
|
||||
- uses: mwilliamson/setup-wabt-action@427f2fdd70bc4dbc2e53c2eb4f19f66162d71bd2 # v4.0.0
|
||||
with:
|
||||
wabt-version: "1.0.36"
|
||||
|
||||
- name: check wasm32-unknown without js
|
||||
run: |
|
||||
cd example_projects/wasm32_without_js/rustpython-without-js
|
||||
@@ -498,6 +712,7 @@ jobs:
|
||||
echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2
|
||||
fi
|
||||
cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm
|
||||
|
||||
- name: build notebook demo
|
||||
if: github.ref == 'refs/heads/release'
|
||||
run: |
|
||||
@@ -507,15 +722,24 @@ jobs:
|
||||
env:
|
||||
NODE_OPTIONS: "--openssl-legacy-provider"
|
||||
working-directory: ./wasm/notebook
|
||||
|
||||
- name: Deploy demo to Github Pages
|
||||
if: success() && github.ref == 'refs/heads/release'
|
||||
uses: peaceiris/actions-gh-pages@v4
|
||||
uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
|
||||
env:
|
||||
ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
|
||||
PUBLISH_DIR: ./wasm/demo/dist
|
||||
EXTERNAL_REPOSITORY: RustPython/demo
|
||||
PUBLISH_BRANCH: master
|
||||
|
||||
- name: Save npm cache
|
||||
# Save only on main or release
|
||||
if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'
|
||||
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ${{ steps.npm-cache-dir.outputs.dir }}
|
||||
key: node-${{ runner.os }}-wasm-demo-${{ hashFiles('wasm/demo/package-lock.json') }}
|
||||
|
||||
wasm-wasi:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
|
||||
name: Run snippets and cpython tests on wasm-wasi
|
||||
@@ -530,12 +754,24 @@ jobs:
|
||||
with:
|
||||
target: wasm32-wasip1
|
||||
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable-wasm32-wasip1-${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable-wasm32-wasip1-
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
|
||||
- name: Setup Wasmer
|
||||
uses: wasmerio/setup-wasmer@v3
|
||||
uses: wasmerio/setup-wasmer@24b15c95293d23f89c68bd40dac76338f773e924 # v3.1
|
||||
|
||||
- name: Install clang
|
||||
uses: ./.github/actions/install-linux-deps
|
||||
@@ -543,8 +779,44 @@ jobs:
|
||||
clang: true
|
||||
|
||||
- name: build rustpython
|
||||
run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose
|
||||
run: cargo build --release --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib,stdio,importlib,host_env --verbose
|
||||
- name: run snippets
|
||||
run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_random.py"
|
||||
- name: run cpython unittest
|
||||
run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/release/rustpython.wasm -- "$(pwd)/Lib/test/test_int.py"
|
||||
|
||||
cargo_doc:
|
||||
needs:
|
||||
- determine_changes
|
||||
if: |
|
||||
(
|
||||
!contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
|
||||
needs.determine_changes.outputs.rust_code == 'true'
|
||||
) || github.ref == 'refs/heads/main'
|
||||
env:
|
||||
RUST_BACKTRACE: full
|
||||
name: cargo doc
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Restore cache
|
||||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
|
||||
restore-keys: |
|
||||
${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}-
|
||||
${{ runner.os }}-stable--
|
||||
|
||||
- name: cargo doc
|
||||
run: cargo doc --locked
|
||||
|
||||
51
.github/workflows/cron-ci.yaml
vendored
51
.github/workflows/cron-ci.yaml
vendored
@@ -1,3 +1,5 @@
|
||||
name: Periodic checks/tasks
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 6"
|
||||
@@ -5,15 +7,14 @@ on:
|
||||
push:
|
||||
paths:
|
||||
- .github/workflows/cron-ci.yaml
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/cron-ci.yaml
|
||||
|
||||
name: Periodic checks/tasks
|
||||
|
||||
env:
|
||||
CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env
|
||||
PYTHON_VERSION: "3.14.3"
|
||||
CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env
|
||||
|
||||
jobs:
|
||||
# codecov collects code coverage data from the rust tests, python snippets and python test suite.
|
||||
@@ -23,30 +24,40 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
# Disable this scheduled job when running on a fork.
|
||||
if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
|
||||
env:
|
||||
INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@cargo-llvm-cov
|
||||
- uses: actions/setup-python@v6.2.0
|
||||
|
||||
- uses: taiki-e/install-action@b550161ef8a7bc4f2a671c0b03a18ac9ccedea1e # v2.79.1
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
tool: cargo-llvm-cov
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
|
||||
- run: sudo apt-get update && sudo apt-get -y install lcov
|
||||
|
||||
- name: Run cargo-llvm-cov with Rust tests.
|
||||
run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env
|
||||
run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env
|
||||
|
||||
- name: Run cargo-llvm-cov with Python snippets.
|
||||
run: python scripts/cargo-llvm-cov.py
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run cargo-llvm-cov with Python test suite.
|
||||
run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed
|
||||
continue-on-error: true
|
||||
|
||||
- name: Prepare code coverage data
|
||||
run: cargo llvm-cov report --lcov --output-path='codecov.lcov'
|
||||
|
||||
- name: Upload to Codecov
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
|
||||
with:
|
||||
files: ./codecov.lcov
|
||||
|
||||
@@ -61,12 +72,15 @@ jobs:
|
||||
persist-credentials: true
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
|
||||
- name: collect tests data
|
||||
run: cargo run --release extra_tests/jsontests.py
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
|
||||
- name: upload tests data to the website
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
env:
|
||||
@@ -96,17 +110,19 @@ jobs:
|
||||
persist-credentials: true
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
|
||||
- name: build rustpython
|
||||
run: cargo build --release --verbose
|
||||
|
||||
- name: Collect what is left data
|
||||
run: |
|
||||
chmod +x ./scripts/whats_left.py
|
||||
./scripts/whats_left.py --features "ssl,sqlite" > whats_left.temp
|
||||
env:
|
||||
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
|
||||
|
||||
- name: Upload data to the website
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
env:
|
||||
@@ -157,16 +173,20 @@ jobs:
|
||||
persist-credentials: true
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v6.2.0
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
|
||||
- run: cargo install cargo-criterion
|
||||
|
||||
- name: build benchmarks
|
||||
run: cargo build --release --benches
|
||||
|
||||
- name: collect execution benchmark data
|
||||
run: cargo criterion --bench execution
|
||||
|
||||
- name: collect microbenchmarks data
|
||||
run: cargo criterion --bench microbenchmarks
|
||||
|
||||
- name: restructure generated files
|
||||
run: |
|
||||
cd ./target/criterion/reports
|
||||
@@ -179,6 +199,7 @@ jobs:
|
||||
cd ..
|
||||
mv reports/* .
|
||||
rmdir reports
|
||||
|
||||
- name: upload benchmark data to the website
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
env:
|
||||
|
||||
88
.github/workflows/lib-deps-check.yaml
vendored
88
.github/workflows/lib-deps-check.yaml
vendored
@@ -10,9 +10,6 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.14.3"
|
||||
|
||||
jobs:
|
||||
check_deps:
|
||||
permissions:
|
||||
@@ -37,46 +34,71 @@ jobs:
|
||||
# Checkout only Lib/ directory from PR head for accurate comparison
|
||||
git checkout ${{ github.event.pull_request.head.sha }} -- Lib/
|
||||
|
||||
- name: Checkout CPython
|
||||
- name: Get target CPython version
|
||||
id: cpython-version
|
||||
run: |
|
||||
git clone --depth 1 --branch "v${{ env.PYTHON_VERSION }}" https://github.com/python/cpython.git cpython
|
||||
version=$(cat .python-version)
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout CPython
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: python/cpython
|
||||
path: cpython
|
||||
ref: "v${{ steps.cpython-version.outputs.version }}"
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get changed Lib files
|
||||
id: changed-files
|
||||
id: all-changed-files
|
||||
run: |
|
||||
# Get the list of changed files under Lib/
|
||||
changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py' | head -50)
|
||||
echo "Changed files:"
|
||||
echo "$changed"
|
||||
{
|
||||
echo 'changed<<EOF'
|
||||
|
||||
# Extract unique module names
|
||||
modules=""
|
||||
for file in $changed; do
|
||||
if [[ "$file" == Lib/test/* ]]; then
|
||||
# Test files: Lib/test/test_pydoc.py -> test_pydoc, Lib/test/test_pydoc/foo.py -> test_pydoc
|
||||
module=$(echo "$file" | sed -E 's|^Lib/test/||; s|\.py$||; s|/.*||')
|
||||
# Skip non-test files in test/ (e.g., support.py, __init__.py)
|
||||
if [[ ! "$module" == test_* ]]; then
|
||||
git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py'
|
||||
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Parse changed files
|
||||
id: changed-files
|
||||
run: |
|
||||
from os import environ
|
||||
|
||||
files = environ["FILES"]
|
||||
modules = set()
|
||||
for file in files.splitlines():
|
||||
file = file.strip()
|
||||
|
||||
if file.startswith("Lib/test/"):
|
||||
# Test files:
|
||||
# Lib/test/test_pydoc.py -> test_pydoc
|
||||
# Lib/test/test_pydoc/foo.py -> test_pydoc
|
||||
module = file.removeprefix("Lib/test/").split("/")[0]
|
||||
if not module.startswith("test_"):
|
||||
continue
|
||||
fi
|
||||
else
|
||||
# Lib files: Lib/foo.py -> foo, Lib/foo/__init__.py -> foo
|
||||
module=$(echo "$file" | sed -E 's|^Lib/||; s|/__init__\.py$||; s|\.py$||; s|/.*||')
|
||||
fi
|
||||
if [[ -n "$module" && ! " $modules " =~ " $module " ]]; then
|
||||
modules="$modules $module"
|
||||
fi
|
||||
done
|
||||
else:
|
||||
# Lib files:
|
||||
# Lib/foo.py -> foo
|
||||
# Lib/foo/__init__.py -> foo
|
||||
module = file.removeprefix("Lib/").split("/")[0]
|
||||
|
||||
module = module.split(".")[0]
|
||||
modules.add(module)
|
||||
|
||||
modules=$(echo "$modules" | xargs) # trim whitespace
|
||||
echo "Detected modules: $modules"
|
||||
echo "modules=$modules" >> $GITHUB_OUTPUT
|
||||
print(f"{modules=}")
|
||||
output = " ".join(sorted(modules))
|
||||
output_file = environ["GITHUB_OUTPUT"]
|
||||
with open(output_file, mode="a", encoding="utf-8") as fd:
|
||||
fd.write(f"modules={output}\n")
|
||||
env:
|
||||
FILES: ${{ steps.all-changed-files.outputs.changed }}
|
||||
shell: python
|
||||
|
||||
- name: Setup Python
|
||||
if: steps.changed-files.outputs.modules != ''
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Run deps check
|
||||
if: steps.changed-files.outputs.modules != ''
|
||||
@@ -92,7 +114,7 @@ jobs:
|
||||
|
||||
- name: Post comment
|
||||
if: steps.deps-check.outputs.deps_output != ''
|
||||
uses: marocchino/sticky-pull-request-comment@v3
|
||||
uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
|
||||
with:
|
||||
header: lib-deps-check
|
||||
number: ${{ github.event.pull_request.number }}
|
||||
@@ -109,7 +131,7 @@ jobs:
|
||||
|
||||
- name: Remove comment if no Lib changes
|
||||
if: steps.changed-files.outputs.modules == ''
|
||||
uses: marocchino/sticky-pull-request-comment@v3
|
||||
uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
|
||||
with:
|
||||
header: lib-deps-check
|
||||
number: ${{ github.event.pull_request.number }}
|
||||
|
||||
22
.github/workflows/release.yml
vendored
22
.github/workflows/release.yml
vendored
@@ -16,11 +16,15 @@ env:
|
||||
X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
|
||||
X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ${{ matrix.os }}
|
||||
# Disable this scheduled job when running on a fork.
|
||||
if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -64,7 +68,7 @@ jobs:
|
||||
libtool: true
|
||||
|
||||
- name: Build RustPython
|
||||
run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls,threading,jit
|
||||
run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls-aws-lc,threading,jit
|
||||
|
||||
- name: Rename Binary
|
||||
run: cp target/${{ matrix.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.target }}
|
||||
@@ -75,7 +79,7 @@ jobs:
|
||||
if: runner.os == 'Windows'
|
||||
|
||||
- name: Upload Binary Artifacts
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustpython-release-${{ runner.os }}-${{ matrix.target }}
|
||||
path: target/rustpython-release-${{ runner.os }}-${{ matrix.target }}*
|
||||
@@ -84,6 +88,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
# Disable this scheduled job when running on a fork.
|
||||
if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
@@ -100,7 +106,7 @@ jobs:
|
||||
run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
|
||||
|
||||
- name: Upload Binary Artifacts
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustpython-release-wasm32-wasip1
|
||||
path: target/rustpython-release-wasm32-wasip1.wasm
|
||||
@@ -108,11 +114,11 @@ jobs:
|
||||
- name: install wasm-pack
|
||||
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
|
||||
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
package-manager-cache: false
|
||||
|
||||
- uses: mwilliamson/setup-wabt-action@febe2a12b7ccb999a6e5d953a8362a3b7ffcf148 # v3.2.0
|
||||
- uses: mwilliamson/setup-wabt-action@427f2fdd70bc4dbc2e53c2eb4f19f66162d71bd2 # v4.0.0
|
||||
with:
|
||||
wabt-version: "1.0.30"
|
||||
|
||||
@@ -135,7 +141,7 @@ jobs:
|
||||
|
||||
- name: Deploy demo to Github Pages
|
||||
if: ${{ github.repository == 'RustPython/RustPython' }}
|
||||
uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
|
||||
uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
|
||||
with:
|
||||
deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }}
|
||||
publish_dir: ./wasm/demo/dist
|
||||
@@ -147,6 +153,8 @@ jobs:
|
||||
# Disable this scheduled job when running on a fork.
|
||||
if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
|
||||
needs: [build, build-wasm]
|
||||
permissions:
|
||||
contents: write # for creating a release
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
@@ -187,7 +195,7 @@ jobs:
|
||||
$PRERELEASE_ARG \
|
||||
bin/rustpython-release-*
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
tag: ${{ github.ref_name }}
|
||||
run: ${{ github.run_number }}
|
||||
PRE_RELEASE_INPUT: ${{ github.event.inputs.pre-release }}
|
||||
|
||||
77
.github/workflows/update-caches.yml
vendored
Normal file
77
.github/workflows/update-caches.yml
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
name: Update Actions Caches
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_PROFILE_TEST_DEBUG: 0
|
||||
CARGO_PROFILE_DEV_DEBUG: 0
|
||||
CARGO_PROFILE_RELEASE_DEBUG: 0
|
||||
CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher
|
||||
|
||||
jobs:
|
||||
build-caches:
|
||||
name: Build Caches
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
toolchain: stable
|
||||
target: ""
|
||||
- os: ubuntu-latest
|
||||
toolchain: stable
|
||||
target: ""
|
||||
- os: windows-latest
|
||||
toolchain: stable
|
||||
target: ""
|
||||
steps:
|
||||
- name: Checkout RustPython main branch
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: RustPython/RustPython
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
|
||||
with:
|
||||
toolchain: ${{ matrix.toolchain }}
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Install macos dependencies
|
||||
uses: ./.github/actions/install-macos-deps
|
||||
with:
|
||||
openssl: true
|
||||
|
||||
- name: Build dev cache # dev profile used by check & doc
|
||||
run: cargo build --profile dev ${{ env.CARGO_ARGS }}
|
||||
|
||||
- name: Build test cache
|
||||
run: cargo build --profile test ${{ env.CARGO_ARGS }}
|
||||
|
||||
- name: Build release cache
|
||||
run: cargo build --profile release ${{ env.CARGO_ARGS }}
|
||||
|
||||
- name: Save cache
|
||||
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }}
|
||||
25
.github/workflows/update-doc-db.yml
vendored
25
.github/workflows/update-doc-db.yml
vendored
@@ -43,7 +43,7 @@ jobs:
|
||||
- name: Generate docs
|
||||
run: python crates/doc/generate.py
|
||||
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: doc-db-${{ inputs.python-version }}-${{ matrix.os }}
|
||||
path: "crates/doc/generated/*.json"
|
||||
@@ -87,19 +87,18 @@ jobs:
|
||||
|
||||
OUTPUT_FILE='crates/doc/src/data.inc.rs'
|
||||
|
||||
echo -n '' > $OUTPUT_FILE
|
||||
# shellcheck disable=SC2016
|
||||
{
|
||||
echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.'
|
||||
echo "// CPython version: ${PYTHON_VERSION}"
|
||||
echo '// spell-checker: disable'
|
||||
echo ''
|
||||
echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {"
|
||||
cat crates/doc/generated/raw_entries.txt
|
||||
echo '};'
|
||||
} > "$OUTPUT_FILE"
|
||||
|
||||
echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE
|
||||
echo "// CPython version: ${PYTHON_VERSION}" >> $OUTPUT_FILE
|
||||
echo '// spell-checker: disable' >> $OUTPUT_FILE
|
||||
|
||||
echo '' >> $OUTPUT_FILE
|
||||
|
||||
echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" >> $OUTPUT_FILE
|
||||
cat crates/doc/generated/raw_entries.txt >> $OUTPUT_FILE
|
||||
echo '};' >> $OUTPUT_FILE
|
||||
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: doc-db-${{ inputs.python-version }}
|
||||
path: "crates/doc/src/data.inc.rs"
|
||||
|
||||
16
.github/workflows/update-libs-status.yaml
vendored
16
.github/workflows/update-libs-status.yaml
vendored
@@ -13,7 +13,6 @@ permissions:
|
||||
issues: write
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "v3.14.3"
|
||||
ISSUE_ID: "6839"
|
||||
|
||||
jobs:
|
||||
@@ -29,13 +28,20 @@ jobs:
|
||||
sparse-checkout: |-
|
||||
Lib
|
||||
scripts/update_lib
|
||||
.python-version
|
||||
|
||||
- name: Clone CPython ${{ env.PYTHON_VERSION }}
|
||||
- name: Get target CPython version
|
||||
id: cpython-version
|
||||
run: |
|
||||
version=$(cat rustpython/.python-version)
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Clone CPython
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: python/cpython
|
||||
path: cpython
|
||||
ref: ${{ env.PYTHON_VERSION }}
|
||||
ref: "v${{ steps.cpython-version.outputs.version }}"
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
Lib
|
||||
@@ -56,14 +62,14 @@ jobs:
|
||||
|
||||
## Summary
|
||||
|
||||
Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ env.PYTHON_VERSION }}\`.
|
||||
Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ steps.cpython-version.outputs.version }}\`.
|
||||
|
||||
Previous versions' issues as reference
|
||||
- 3.13: #5529
|
||||
|
||||
<!--
|
||||
Quick guideline for Copilot:
|
||||
# Clone \`github.com/python/cpython\` \`${{ env.PYTHON_VERSION }}\` tag under RustPython working dir with depth 1 option; never 3.14.0 or 3.14.1 or 3.14.2
|
||||
# Clone \`github.com/python/cpython\` \`${{ steps.cpython-version.outputs.version }}\` tag under RustPython working dir with depth 1 option; never 3.14.0 or 3.14.1 or 3.14.2
|
||||
# Pick a library or test to update. Probably user give one.
|
||||
# Run \`python3 scripts/update_lib quick <name>\`
|
||||
# A commit is automatically created. push the commit.
|
||||
|
||||
76
.github/workflows/upgrade-pylib.lock.yml
generated
vendored
76
.github/workflows/upgrade-pylib.lock.yml
generated
vendored
@@ -58,11 +58,11 @@ jobs:
|
||||
comment_repo: ""
|
||||
steps:
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5
|
||||
uses: github/gh-aw/actions/setup@2c1a237d2048b0e2412e7d7528892ea1257840e2 # v0.74.4
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Check workflow file timestamps
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_WORKFLOW_FILE: "upgrade-pylib.lock.yml"
|
||||
with:
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
|
||||
steps:
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5
|
||||
uses: github/gh-aw/actions/setup@2c1a237d2048b0e2412e7d7528892ea1257840e2 # v0.74.4
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Checkout repository
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh
|
||||
# Cache configuration from frontmatter processed below
|
||||
- name: Cache (cpython-lib-${{ env.PYTHON_VERSION }})
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
key: cpython-lib-${{ env.PYTHON_VERSION }}
|
||||
path: cpython
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
id: checkout-pr
|
||||
if: |
|
||||
github.event.pull_request
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
@@ -147,7 +147,7 @@ jobs:
|
||||
await main();
|
||||
- name: Generate agentic run info
|
||||
id: generate_aw_info
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
@@ -201,7 +201,7 @@ jobs:
|
||||
run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.16.4
|
||||
- name: Determine automatic lockdown mode for GitHub MCP server
|
||||
id: determine-automatic-lockdown
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
|
||||
GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
|
||||
@@ -484,7 +484,7 @@ jobs:
|
||||
}
|
||||
GH_AW_MCP_CONFIG_EOF
|
||||
- name: Generate workflow overview
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs');
|
||||
@@ -508,10 +508,11 @@ jobs:
|
||||
cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT"
|
||||
<system>
|
||||
GH_AW_PROMPT_EOF
|
||||
cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT"
|
||||
cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT"
|
||||
cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT"
|
||||
cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT"
|
||||
{
|
||||
cat "/opt/gh-aw/prompts/xpia.md"
|
||||
cat "/opt/gh-aw/prompts/temp_folder_prompt.md"
|
||||
cat "/opt/gh-aw/prompts/markdown.md"
|
||||
cat << 'GH_AW_PROMPT_EOF'
|
||||
<safe-outputs>
|
||||
<description>GitHub API Access Instructions</description>
|
||||
<important>
|
||||
@@ -569,14 +570,15 @@ jobs:
|
||||
</github-context>
|
||||
|
||||
GH_AW_PROMPT_EOF
|
||||
cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT"
|
||||
cat << 'GH_AW_PROMPT_EOF'
|
||||
</system>
|
||||
GH_AW_PROMPT_EOF
|
||||
cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT"
|
||||
cat << 'GH_AW_PROMPT_EOF'
|
||||
{{#runtime-import .github/workflows/upgrade-pylib.md}}
|
||||
GH_AW_PROMPT_EOF
|
||||
} >> "$GH_AW_PROMPT"
|
||||
- name: Substitute placeholders
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
|
||||
GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }}
|
||||
@@ -610,7 +612,7 @@ jobs:
|
||||
}
|
||||
});
|
||||
- name: Interpolate variables and render templates
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
|
||||
GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }}
|
||||
@@ -690,7 +692,7 @@ jobs:
|
||||
bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID"
|
||||
- name: Redact secrets in logs
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs');
|
||||
@@ -705,14 +707,14 @@ jobs:
|
||||
SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Upload Safe Outputs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: safe-output
|
||||
path: ${{ env.GH_AW_SAFE_OUTPUTS }}
|
||||
if-no-files-found: warn
|
||||
- name: Ingest agent output
|
||||
id: collect_output
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }}
|
||||
GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com"
|
||||
@@ -726,13 +728,13 @@ jobs:
|
||||
await main();
|
||||
- name: Upload sanitized agent output
|
||||
if: always() && env.GH_AW_AGENT_OUTPUT
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: agent-output
|
||||
path: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
if-no-files-found: warn
|
||||
- name: Upload engine output files
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: agent_outputs
|
||||
path: |
|
||||
@@ -741,7 +743,7 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
- name: Parse agent logs for step summary
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/
|
||||
with:
|
||||
@@ -752,7 +754,7 @@ jobs:
|
||||
await main();
|
||||
- name: Parse MCP gateway logs for step summary
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs');
|
||||
@@ -772,7 +774,7 @@ jobs:
|
||||
- name: Upload agent artifacts
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: agent-artifacts
|
||||
path: |
|
||||
@@ -804,7 +806,7 @@ jobs:
|
||||
total_count: ${{ steps.missing_tool.outputs.total_count }}
|
||||
steps:
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5
|
||||
uses: github/gh-aw/actions/setup@2c1a237d2048b0e2412e7d7528892ea1257840e2 # v0.74.4
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
@@ -820,7 +822,7 @@ jobs:
|
||||
echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV"
|
||||
- name: Process No-Op Messages
|
||||
id: noop
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_NOOP_MAX: 1
|
||||
@@ -834,7 +836,7 @@ jobs:
|
||||
await main();
|
||||
- name: Record Missing Tool
|
||||
id: missing_tool
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_WORKFLOW_NAME: "Upgrade Python Library"
|
||||
@@ -847,7 +849,7 @@ jobs:
|
||||
await main();
|
||||
- name: Handle Agent Failure
|
||||
id: handle_agent_failure
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_WORKFLOW_NAME: "Upgrade Python Library"
|
||||
@@ -865,7 +867,7 @@ jobs:
|
||||
await main();
|
||||
- name: Handle No-Op Message
|
||||
id: handle_noop_message
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_WORKFLOW_NAME: "Upgrade Python Library"
|
||||
@@ -882,7 +884,7 @@ jobs:
|
||||
await main();
|
||||
- name: Handle Create Pull Request Error
|
||||
id: handle_create_pr_error
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_WORKFLOW_NAME: "Upgrade Python Library"
|
||||
@@ -896,7 +898,7 @@ jobs:
|
||||
await main();
|
||||
- name: Update reaction comment with completion status
|
||||
id: conclusion
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}
|
||||
@@ -925,7 +927,7 @@ jobs:
|
||||
success: ${{ steps.parse_results.outputs.success }}
|
||||
steps:
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5
|
||||
uses: github/gh-aw/actions/setup@2c1a237d2048b0e2412e7d7528892ea1257840e2 # v0.74.4
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent artifacts
|
||||
@@ -946,7 +948,7 @@ jobs:
|
||||
run: |
|
||||
echo "Agent output-types: $AGENT_OUTPUT_TYPES"
|
||||
- name: Setup threat detection
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
WORKFLOW_NAME: "Upgrade Python Library"
|
||||
WORKFLOW_DESCRIPTION: "Pick an out-of-sync Python library from the todo list and upgrade it\nby running `scripts/update_lib quick`, then open a pull request."
|
||||
@@ -999,7 +1001,7 @@ jobs:
|
||||
XDG_CONFIG_HOME: /home/runner
|
||||
- name: Parse threat detection results
|
||||
id: parse_results
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs');
|
||||
@@ -1008,7 +1010,7 @@ jobs:
|
||||
await main();
|
||||
- name: Upload threat detection log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: threat-detection.log
|
||||
path: /tmp/gh-aw/threat-detection/detection.log
|
||||
@@ -1037,7 +1039,7 @@ jobs:
|
||||
process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
|
||||
steps:
|
||||
- name: Setup Scripts
|
||||
uses: github/gh-aw/actions/setup@48d8fdfddc8cad854ac0c70ceb573f09fb8f9c9b # v0.62.5
|
||||
uses: github/gh-aw/actions/setup@2c1a237d2048b0e2412e7d7528892ea1257840e2 # v0.74.4
|
||||
with:
|
||||
destination: /opt/gh-aw/actions
|
||||
- name: Download agent output artifact
|
||||
@@ -1079,7 +1081,7 @@ jobs:
|
||||
echo "Git configured with standard GitHub Actions identity"
|
||||
- name: Process Safe Outputs
|
||||
id: process_safe_outputs
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }}
|
||||
GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_pull_request\":{\"base_branch\":\"${{ github.ref_name }}\",\"draft\":false,\"expires\":30,\"labels\":[\"pylib-sync\"],\"max\":1,\"max_patch_size\":1024,\"title_prefix\":\"Update \"},\"missing_data\":{},\"missing_tool\":{}}"
|
||||
|
||||
2
.github/workflows/upgrade-pylib.md
vendored
2
.github/workflows/upgrade-pylib.md
vendored
@@ -52,7 +52,7 @@ cache:
|
||||
- cpython-lib-
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "v3.14.3"
|
||||
PYTHON_VERSION: "v3.14.4"
|
||||
ISSUE_ID: "6839"
|
||||
---
|
||||
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -10,7 +10,6 @@ __pycache__/
|
||||
wasm-pack.log
|
||||
.idea/
|
||||
.envrc
|
||||
.python-version
|
||||
|
||||
flame-graph.html
|
||||
flame.txt
|
||||
@@ -28,4 +27,4 @@ Lib/site-packages/*
|
||||
Lib/test/data/*
|
||||
!Lib/test/data/README
|
||||
cpython/
|
||||
|
||||
.claude/scheduled_tasks.lock
|
||||
@@ -10,7 +10,7 @@ repos:
|
||||
priority: 0
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.7
|
||||
rev: v0.15.12
|
||||
hooks:
|
||||
- id: ruff-format
|
||||
priority: 0
|
||||
@@ -40,17 +40,30 @@ repos:
|
||||
types: [rust]
|
||||
priority: 0
|
||||
|
||||
- id: generate-opcode-metadata
|
||||
name: generate opcode metadata
|
||||
entry: python scripts/generate_opcode_metadata.py
|
||||
files: '^(crates/compiler-core/src/bytecode/instruction\.rs|scripts/generate_opcode_metadata\.py)$'
|
||||
- id: generate-rs-opcode-metadata
|
||||
name: generate rust opcode metadata
|
||||
entry: python tools/opcode_metadata/generate_rs_opcode_metadata.py
|
||||
files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
|
||||
pass_filenames: false
|
||||
language: system
|
||||
require_serial: true
|
||||
priority: 1 # so rustfmt runs first
|
||||
stages:
|
||||
- manual
|
||||
|
||||
- id: generate-py-opcode-metadata
|
||||
name: generate python opcode metadata
|
||||
entry: python tools/opcode_metadata/generate_py_opcode_metadata.py
|
||||
files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
|
||||
pass_filenames: false
|
||||
language: system
|
||||
require_serial: true
|
||||
priority: 1 # so rustfmt runs first
|
||||
stages:
|
||||
- manual
|
||||
|
||||
- repo: https://github.com/streetsidesoftware/cspell-cli
|
||||
rev: v9.7.0
|
||||
rev: v10.0.0
|
||||
hooks:
|
||||
- id: cspell
|
||||
types: [rust]
|
||||
@@ -64,7 +77,7 @@ repos:
|
||||
priority: 0
|
||||
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: v3.8.1
|
||||
rev: v3.8.3
|
||||
hooks:
|
||||
- id: prettier
|
||||
files: '^wasm/.*$'
|
||||
|
||||
1
.python-version
Normal file
1
.python-version
Normal file
@@ -0,0 +1 @@
|
||||
3.14.5
|
||||
41
AGENTS.md
41
AGENTS.md
@@ -38,6 +38,12 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0
|
||||
- Always ask the user before performing any git operations that affect the remote repository
|
||||
- Commits can be created locally when requested, but pushing and PR creation require explicit approval
|
||||
|
||||
**CRITICAL: Pre-commit Checks**
|
||||
- Before creating ANY commit, you MUST run `prek run --all-files` (or `pre-commit run --all-files`) AND the full test suite. Both must pass — do not commit if either fails.
|
||||
- Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test <module>` modules.
|
||||
- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes, re-run `prek` until it reports a clean pass, then re-run the tests, then commit.
|
||||
- NEVER bypass these checks with `--no-verify`, `--no-gpg-sign`, or by skipping tests "because the change is small". If a hook or test fails, fix the underlying issue and create a new commit — do not amend or force the failing commit through.
|
||||
|
||||
## Important Development Notes
|
||||
|
||||
### Running Python Code
|
||||
@@ -81,6 +87,35 @@ The `Lib/` directory contains Python standard library files copied from the CPyt
|
||||
- `unittest.skip("TODO: RustPython <reason>")`
|
||||
- `unittest.expectedFailure` with `# TODO: RUSTPYTHON <reason>` comment
|
||||
|
||||
#### Choosing the right marker
|
||||
|
||||
When marking a test that fails on RustPython, prefer one of the following forms:
|
||||
|
||||
```python
|
||||
@unittest.expectedFailure # TODO: RUSTPYTHON; <reason>
|
||||
# or
|
||||
@unittest.expectedFailureIf(<condition>, "TODO: RUSTPYTHON; <reason>")
|
||||
```
|
||||
|
||||
If the test would crash the interpreter (segfault, Rust panic, abort, infinite loop), use `skip` instead so the rest of the suite can still run:
|
||||
|
||||
```python
|
||||
@unittest.skip("TODO: RUSTPYTHON; <reason>")
|
||||
# or
|
||||
@unittest.skipIf(<condition>, "TODO: RUSTPYTHON; <reason>")
|
||||
```
|
||||
|
||||
**When to use which:**
|
||||
|
||||
- **Prefer `expectedFailure` / `expectedFailureIf`** by default. The test body still runs, so if RustPython is later fixed, the unexpected pass surfaces immediately and the decorator can be removed. Use the conditional `*If` form when the failure is environment-specific (e.g., a platform or build flag).
|
||||
- **Use `skip` / `skipIf` only when running the test would take down the test process** — segfaults, Rust panics, aborts, or hangs that block subsequent tests. Skipping keeps the suite usable; `expectedFailure` cannot help here, because the test body still executes.
|
||||
|
||||
To find WIP entries that are partly modified and may need follow-up:
|
||||
|
||||
```bash
|
||||
grep -d recurse 'TODO: RUSTPYTHON' Lib/test/
|
||||
```
|
||||
|
||||
### Clean Build
|
||||
|
||||
When you modify bytecode instructions, a full clean is required:
|
||||
@@ -129,6 +164,7 @@ Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is h
|
||||
|
||||
- Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code.
|
||||
- Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value.
|
||||
- Do not put `///` doc comments on items annotated with `#[pyattr]`, `#[pyclass]`, or `#[pyfunction]`. The derive macros pull authoritative docstrings from CPython via the `rustpython-doc` crate; a Rust doc comment overrides that source, and on `#[pyattr]` it is silently dropped.
|
||||
|
||||
#### Avoid Duplicate Code in Branches
|
||||
|
||||
@@ -258,9 +294,14 @@ See DEVELOPMENT.md "CPython Version Upgrade Checklist" section.
|
||||
- Document that it requires PEP 695 support
|
||||
- Focus on tests that can be fixed through Rust code changes only
|
||||
|
||||
## CI Workflows
|
||||
|
||||
If you modify any file under `.github/workflows/`, the change must pass a [zizmor](https://docs.zizmor.sh/) scan in CI.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Check the [architecture document](/architecture/architecture.md) for a high-level overview
|
||||
- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions
|
||||
- Generate documentation with `cargo doc --no-deps --all`
|
||||
- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)
|
||||
- [How to update test files](https://github.com/RustPython/RustPython/wiki/How-to-update-test-files#checkout-cpython-source-code-initial-setup) — guide for syncing test cases from upstream CPython into the `Lib/` directory
|
||||
|
||||
@@ -1,4 +1,28 @@
|
||||
# RustPython Development Guide and Tips
|
||||
# Contributing to RustPython
|
||||
|
||||
Contributions are more than welcome, and in many cases we are happy to guide
|
||||
contributors through PRs or on [**Discord**](https://discord.gg/vru8NypEhv).
|
||||
|
||||
## Finding ways to help
|
||||
|
||||
We label issues that would be good for a first time contributor as [`good first issue`](https://github.com/RustPython/RustPython/issues?q=label%3A%22good+first+issue%22+is%3Aissue+is%3Aopen+).
|
||||
Also checkout the [issue tracker](https://github.com/RustPython/RustPython/issues) for all open issues.
|
||||
|
||||
You can enhance CPython compatibility by increasing our unittest coverage, you can see [This pinned issue](https://github.com/RustPython/RustPython/issues/6839) to see which libs and tests need be updated to our current supported python version.
|
||||
|
||||
Another approach is to checkout the source code: builtin functions and object
|
||||
methods are often the simplest and easiest way to contribute.
|
||||
|
||||
You can also simply run `python -I scripts/whats_left.py` to assist in finding any unimplemented method.
|
||||
|
||||
## Use of AI
|
||||
|
||||
We **require all use of AI in contributions to follow our
|
||||
[AI Policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md)**.
|
||||
|
||||
If your contribution does not follow the policy, it will be closed.
|
||||
|
||||
## RustPython Development Guide and Tips
|
||||
|
||||
RustPython attracts developers with interest and experience in Rust, Python,
|
||||
or WebAssembly. Whether you are familiar with Rust, Python, or
|
||||
1688
Cargo.lock
generated
1688
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
188
Cargo.toml
188
Cargo.toml
@@ -10,7 +10,8 @@ repository.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"]
|
||||
capi = ["dep:rustpython-capi", "threading"]
|
||||
default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls-aws-lc", "host_env"]
|
||||
host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"]
|
||||
importlib = ["rustpython-vm/importlib"]
|
||||
encodings = ["rustpython-vm/encodings"]
|
||||
@@ -21,30 +22,35 @@ freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/fre
|
||||
jit = ["rustpython-vm/jit"]
|
||||
threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
|
||||
sqlite = ["rustpython-stdlib/sqlite"]
|
||||
ssl = []
|
||||
ssl = ["host_env"]
|
||||
ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"]
|
||||
ssl-rustls-aws-lc = ["ssl-rustls", "dep:rustls", "rustls/aws_lc_rs"]
|
||||
ssl-rustls-aws-lc-fips = ["ssl-rustls-aws-lc", "rustls/fips"]
|
||||
ssl-openssl = ["ssl", "rustpython-stdlib/ssl-openssl"]
|
||||
ssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-vendor"]
|
||||
ssl-openssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-openssl-vendor"]
|
||||
tkinter = ["rustpython-stdlib/tkinter"]
|
||||
|
||||
[build-dependencies]
|
||||
winresource = "0.1"
|
||||
|
||||
[dependencies]
|
||||
rustpython-capi = { workspace = true, optional = true }
|
||||
rustpython-compiler = { workspace = true }
|
||||
rustpython-pylib = { workspace = true, optional = true }
|
||||
rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
|
||||
rustpython-vm = { workspace = true, features = ["compiler", "gc"] }
|
||||
|
||||
cfg-if = { workspace = true }
|
||||
log = { workspace = true }
|
||||
flame = { workspace = true, optional = true }
|
||||
|
||||
lexopt = "0.3"
|
||||
dirs = { package = "dirs-next", version = "2.0" }
|
||||
dirs = "6"
|
||||
env_logger = "0.11"
|
||||
flamescope = { version = "0.1.2", optional = true }
|
||||
|
||||
rustls = { workspace = true, optional = true }
|
||||
rustls-graviola = { workspace = true, optional = true }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
libc = { workspace = true }
|
||||
|
||||
@@ -53,7 +59,7 @@ rustyline = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
pyo3 = { version = "0.28.2", features = ["auto-initialize"] }
|
||||
pyo3 = { workspace = true, features = ["auto-initialize"] }
|
||||
rustpython-stdlib = { workspace = true }
|
||||
ruff_python_parser = { workspace = true }
|
||||
|
||||
@@ -69,6 +75,17 @@ harness = false
|
||||
name = "rustpython"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[example]]
|
||||
name = "custom_tls_providers"
|
||||
path = "examples/custom_tls_providers.rs"
|
||||
required-features = [
|
||||
"rustls-graviola",
|
||||
"rustls/ring",
|
||||
"rustpython-pylib/freeze-stdlib",
|
||||
"rustpython-stdlib/ssl-rustls",
|
||||
"rustpython-vm/freeze-stdlib",
|
||||
]
|
||||
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
@@ -136,15 +153,17 @@ exclude = ["pymath"]
|
||||
version = "0.5.0"
|
||||
authors = ["RustPython Team"]
|
||||
edition = "2024"
|
||||
rust-version = "1.93.0"
|
||||
rust-version = "1.95.0"
|
||||
repository = "https://github.com/RustPython/RustPython"
|
||||
license = "MIT"
|
||||
|
||||
[workspace.dependencies]
|
||||
rustpython-capi = { path = "crates/capi", version = "0.5.0" }
|
||||
rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" }
|
||||
rustpython-compiler = { path = "crates/compiler", version = "0.5.0" }
|
||||
rustpython-codegen = { path = "crates/codegen", version = "0.5.0" }
|
||||
rustpython-common = { path = "crates/common", version = "0.5.0" }
|
||||
rustpython-host_env = { path = "crates/host_env", version = "0.5.0" }
|
||||
rustpython-derive = { path = "crates/derive", version = "0.5.0" }
|
||||
rustpython-derive-impl = { path = "crates/derive-impl", version = "0.5.0" }
|
||||
rustpython-jit = { path = "crates/jit", version = "0.5.0" }
|
||||
@@ -170,66 +189,141 @@ ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.15.8"
|
||||
# ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
|
||||
# ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
|
||||
|
||||
der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] }
|
||||
phf = { version = "0.13.1", default-features = false, features = ["macros"]}
|
||||
ahash = "0.8.12"
|
||||
adler32 = "1.2.0"
|
||||
approx = "0.5.1"
|
||||
ascii = "1.1"
|
||||
base64 = "0.22"
|
||||
blake2 = "0.10.4"
|
||||
bitflags = "2.11.0"
|
||||
bitflagset = "0.0.3"
|
||||
bstr = "1"
|
||||
cfg-if = "1.0"
|
||||
chrono = { version = "0.4.44", default-features = false, features = ["clock", "oldtime", "std"] }
|
||||
bzip2 = "0.6"
|
||||
chrono = { version = "0.4.44", default-features = false, features = ["clock", "std"] }
|
||||
console_error_panic_hook = "0.1"
|
||||
constant_time_eq = "0.4"
|
||||
cranelift = "0.131.2"
|
||||
cranelift-jit = "0.131.2"
|
||||
cranelift-module = "0.131.0"
|
||||
crc32fast = "1.3.2"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
crossbeam-utils = "0.8.21"
|
||||
csv-core = "0.1.11"
|
||||
digest = "0.10.7"
|
||||
dns-lookup = "3.0"
|
||||
dyn-clone = "1.0.10"
|
||||
exitcode = "1.1.2"
|
||||
flame = "0.2.2"
|
||||
flamer = "0.5"
|
||||
flate2 = { version = "1.1.9", default-features = false }
|
||||
# Bump only when the openssl crate bumps it
|
||||
foreign-types-shared = "0.1"
|
||||
gethostname = "1.0.2"
|
||||
getrandom = { version = "0.3", features = ["std"] }
|
||||
glob = "0.3"
|
||||
half = "2"
|
||||
hex = "0.4.3"
|
||||
indexmap = { version = "2.13.0", features = ["std"] }
|
||||
insta = "1.46"
|
||||
hexf-parse = "0.2.1"
|
||||
hmac = "0.12"
|
||||
indexmap = { version = "2.14.0", features = ["std"] }
|
||||
insta = "1.47"
|
||||
itertools = "0.14.0"
|
||||
is-macro = "0.3.7"
|
||||
js-sys = "0.3"
|
||||
junction = "1.4.2"
|
||||
libc = "0.2.183"
|
||||
lexical-parse-float = "1.0.6"
|
||||
libc = "0.2.186"
|
||||
libffi = "5"
|
||||
libloading = "0.9"
|
||||
liblzma = "0.4"
|
||||
liblzma-sys = "0.4"
|
||||
libsqlite3-sys = "0.37"
|
||||
libz-rs-sys = "0.6"
|
||||
lock_api = "0.4"
|
||||
log = "0.4.29"
|
||||
nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
|
||||
lz4_flex = "0.13"
|
||||
nix = { version = "0.31", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
|
||||
mac_address = "1.1.3"
|
||||
malachite-bigint = "0.9.1"
|
||||
malachite-q = "0.9.1"
|
||||
malachite-base = "0.9.1"
|
||||
md-5 = "0.10.1"
|
||||
memchr = "2.8.0"
|
||||
memmap2 = "0.9.10"
|
||||
mt19937 = "<=3.2" # upgrade it once rand is upgraded
|
||||
num-complex = "0.4.6"
|
||||
num-integer = "0.1.46"
|
||||
num-traits = "0.2"
|
||||
num_cpus = "1.17.0"
|
||||
num_enum = { version = "0.7", default-features = false }
|
||||
oid-registry = "0.8"
|
||||
openssl = "0.10.80"
|
||||
openssl-sys = "0.9.110"
|
||||
openssl-probe = "0.2.1"
|
||||
optional = "0.5"
|
||||
parking_lot = "0.12.3"
|
||||
paste = "1.0.15"
|
||||
pbkdf2 = "0.12"
|
||||
pem-rfc7468 = "1.0"
|
||||
pkcs8 = "0.11"
|
||||
proc-macro2 = "1.0.105"
|
||||
psm = "0.1"
|
||||
pymath = { version = "0.2.0", features = ["mul_add", "malachite-bigint", "complex"] }
|
||||
pyo3 = "0.28"
|
||||
quote = "1.0.45"
|
||||
radium = "1.1.1"
|
||||
rand = "0.9"
|
||||
rand_core = { version = "0.9", features = ["os_rng"] }
|
||||
rustix = { version = "1.1", features = ["event"] }
|
||||
rustyline = "17.0.1"
|
||||
rapidhash = "4.4.1"
|
||||
result-like = "0.5.0"
|
||||
rustix = { version = "1.1", features = ["event", "param", "system"] }
|
||||
rustls = { version = "0.23.39", default-features = false }
|
||||
rustls-graviola = "0.3"
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pemfile = "2.2"
|
||||
rustls-platform-verifier = "0.7"
|
||||
rustyline = "18"
|
||||
serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] }
|
||||
schannel = "0.1.29"
|
||||
scoped-tls = "1"
|
||||
scopeguard = "1"
|
||||
serde-wasm-bindgen = "0.6.5"
|
||||
sha-1 = "0.10.0"
|
||||
sha2 = "0.10.2"
|
||||
sha3 = "0.10.1"
|
||||
siphasher = "1"
|
||||
socket2 = "0.6.3"
|
||||
static_assertions = "1.1"
|
||||
strum = "0.28"
|
||||
strum_macros = "0.28"
|
||||
syn = "2"
|
||||
syn-ext = "0.5.0"
|
||||
system-configuration = "0.7.0"
|
||||
tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" }
|
||||
textwrap = { version = "0.16.2", default-features = false }
|
||||
termios = "0.3.3"
|
||||
thiserror = "2.0"
|
||||
timsort = "0.1.2"
|
||||
tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" }
|
||||
icu_casemap = "2"
|
||||
icu_locale = "2"
|
||||
icu_properties = "2"
|
||||
icu_normalizer = "2"
|
||||
unicode-casing = "0.1.1"
|
||||
uuid = "1.23.1"
|
||||
ucd = "0.1.1"
|
||||
unic-ucd-age = "0.9.0"
|
||||
unicode_names2 = "2.0.0"
|
||||
widestring = "1.2.0"
|
||||
windows-sys = "0.61.2"
|
||||
wasm-bindgen = "0.2.106"
|
||||
wasm-bindgen-futures = "0.4"
|
||||
web-sys = "0.3"
|
||||
webpki-roots = "1.0"
|
||||
which = "8"
|
||||
x509-cert = "0.2.5"
|
||||
x509-parser = "0.18"
|
||||
xml = "1.2"
|
||||
writeable = "0.6"
|
||||
|
||||
# Lints
|
||||
|
||||
@@ -237,13 +331,61 @@ wasm-bindgen = "0.2.106"
|
||||
unsafe_code = "allow"
|
||||
unsafe_op_in_unsafe_fn = "deny"
|
||||
elided_lifetimes_in_paths = "warn"
|
||||
unreachable_pub = "warn"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
correctness = { level = "warn", priority = -2 }
|
||||
suspicious = { level = "warn", priority = -2 }
|
||||
perf = { level = "warn", priority = -2 }
|
||||
style = { level = "warn", priority = -2 }
|
||||
complexity = { level = "warn", priority = -2 }
|
||||
# pedantic = { level = "warn", priority = -2 } # TODO: Enable this
|
||||
|
||||
missing_errors_doc = "allow" # Too many errors. No auto-fix available
|
||||
missing_panics_doc = "allow" # Too many errors. No auto-fix available
|
||||
match_same_arms = "allow" # Not always more readable
|
||||
if_not_else = "allow" # Not always more readable
|
||||
single_match_else = "allow"
|
||||
similar_names = "allow"
|
||||
|
||||
# restriction lints
|
||||
alloc_instead_of_core = "warn"
|
||||
cfg_not_test = "warn"
|
||||
redundant_test_prefix = "warn"
|
||||
std_instead_of_alloc = "warn"
|
||||
std_instead_of_core = "warn"
|
||||
perf = "warn"
|
||||
style = "warn"
|
||||
complexity = "warn"
|
||||
suspicious = "warn"
|
||||
correctness = "warn"
|
||||
tests_outside_test_module = "warn"
|
||||
|
||||
# nursery lints to enforce gradually
|
||||
debug_assert_with_mut_call = "warn"
|
||||
derive_partial_eq_without_eq = "warn"
|
||||
imprecise_flops = "warn"
|
||||
or_fun_call = "warn"
|
||||
redundant_clone = "warn"
|
||||
search_is_some = "warn"
|
||||
single_option_map = "warn"
|
||||
trait_duplication_in_bounds = "warn"
|
||||
unused_peekable = "warn"
|
||||
unused_rounding = "warn"
|
||||
use_self = "warn"
|
||||
useless_let_if_seq = "warn"
|
||||
|
||||
# pedantic lints to enforce gradually
|
||||
cloned_instead_of_copied = "warn"
|
||||
collapsible_else_if = "warn"
|
||||
comparison_chain = "warn"
|
||||
explicit_into_iter_loop = "warn"
|
||||
explicit_iter_loop = "warn"
|
||||
filter_map_next = "warn"
|
||||
flat_map_option = "warn"
|
||||
format_collect = "warn"
|
||||
from_iter_instead_of_collect = "warn"
|
||||
inconsistent_struct_constructor = "warn"
|
||||
inefficient_to_string = "warn"
|
||||
manual_is_variant_and = "warn"
|
||||
map_unwrap_or = "warn"
|
||||
must_use_candidate = "warn"
|
||||
redundant_else = "warn"
|
||||
uninlined_format_args = "warn"
|
||||
unnecessary_wraps = "warn"
|
||||
unnested_or_patterns = "warn"
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 RustPython Team
|
||||
Copyright (c) 2026 RustPython Team
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
7
Lib/_android_support.py
vendored
7
Lib/_android_support.py
vendored
@@ -168,6 +168,13 @@ class Logcat:
|
||||
# message.
|
||||
message = message.replace(b"\x00", b"\xc0\x80")
|
||||
|
||||
# On API level 30 and higher, Logcat will strip any number of leading
|
||||
# newlines. This is visible in all `logcat` modes, even --binary. Work
|
||||
# around this by adding a leading space, which shouldn't make any
|
||||
# difference to the log's usability.
|
||||
if message.startswith(b"\n"):
|
||||
message = b" " + message
|
||||
|
||||
with self._lock:
|
||||
now = time()
|
||||
self._bucket_level += (
|
||||
|
||||
2
Lib/_opcode_metadata.py
generated
vendored
2
Lib/_opcode_metadata.py
generated
vendored
@@ -1,4 +1,4 @@
|
||||
# This file is generated by scripts/generate_opcode_metadata.py
|
||||
# This file is generated by tools/opcode_metadata/generate_py_opcode_metadata.py
|
||||
# for RustPython bytecode format (CPython 3.14 compatible opcode numbers).
|
||||
# Do not edit!
|
||||
|
||||
|
||||
55
Lib/annotationlib.py
vendored
55
Lib/annotationlib.py
vendored
@@ -47,6 +47,7 @@ _SLOTS = (
|
||||
"__cell__",
|
||||
"__owner__",
|
||||
"__stringifier_dict__",
|
||||
"__resolved_str_cache__",
|
||||
)
|
||||
|
||||
|
||||
@@ -94,6 +95,7 @@ class ForwardRef:
|
||||
# value later.
|
||||
self.__code__ = None
|
||||
self.__ast_node__ = None
|
||||
self.__resolved_str_cache__ = None
|
||||
|
||||
def __init_subclass__(cls, /, *args, **kwds):
|
||||
raise TypeError("Cannot subclass ForwardRef")
|
||||
@@ -113,7 +115,7 @@ class ForwardRef:
|
||||
"""
|
||||
match format:
|
||||
case Format.STRING:
|
||||
return self.__forward_arg__
|
||||
return self.__resolved_str__
|
||||
case Format.VALUE:
|
||||
is_forwardref_format = False
|
||||
case Format.FORWARDREF:
|
||||
@@ -258,6 +260,24 @@ class ForwardRef:
|
||||
"Attempted to access '__forward_arg__' on an uninitialized ForwardRef"
|
||||
)
|
||||
|
||||
@property
|
||||
def __resolved_str__(self):
|
||||
# __forward_arg__ with any names from __extra_names__ replaced
|
||||
# with the type_repr of the value they represent
|
||||
if self.__resolved_str_cache__ is None:
|
||||
resolved_str = self.__forward_arg__
|
||||
names = self.__extra_names__
|
||||
|
||||
if names:
|
||||
visitor = _ExtraNameFixer(names)
|
||||
ast_expr = ast.parse(resolved_str, mode="eval").body
|
||||
node = visitor.visit(ast_expr)
|
||||
resolved_str = ast.unparse(node)
|
||||
|
||||
self.__resolved_str_cache__ = resolved_str
|
||||
|
||||
return self.__resolved_str_cache__
|
||||
|
||||
@property
|
||||
def __forward_code__(self):
|
||||
if self.__code__ is not None:
|
||||
@@ -321,7 +341,7 @@ class ForwardRef:
|
||||
extra.append(", is_class=True")
|
||||
if self.__owner__ is not None:
|
||||
extra.append(f", owner={self.__owner__!r}")
|
||||
return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})"
|
||||
return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})"
|
||||
|
||||
|
||||
_Template = type(t"")
|
||||
@@ -357,6 +377,7 @@ class _Stringifier:
|
||||
self.__cell__ = cell
|
||||
self.__owner__ = owner
|
||||
self.__stringifier_dict__ = stringifier_dict
|
||||
self.__resolved_str_cache__ = None # Needed for ForwardRef
|
||||
|
||||
def __convert_to_ast(self, other):
|
||||
if isinstance(other, _Stringifier):
|
||||
@@ -919,7 +940,7 @@ def get_annotations(
|
||||
does not exist, the __annotate__ function is called. The
|
||||
FORWARDREF format uses __annotations__ if it exists and can be
|
||||
evaluated, and otherwise falls back to calling the __annotate__ function.
|
||||
The SOURCE format tries __annotate__ first, and falls back to
|
||||
The STRING format tries __annotate__ first, and falls back to
|
||||
using __annotations__, stringified using annotations_to_string().
|
||||
|
||||
This function handles several details for you:
|
||||
@@ -1037,13 +1058,26 @@ def get_annotations(
|
||||
obj_globals = obj_locals = unwrap = None
|
||||
|
||||
if unwrap is not None:
|
||||
# Use an id-based visited set to detect cycles in the __wrapped__
|
||||
# and functools.partial.func chain (e.g. f.__wrapped__ = f).
|
||||
# On cycle detection we stop and use whatever __globals__ we have
|
||||
# found so far, mirroring the approach of inspect.unwrap().
|
||||
_seen_ids = {id(unwrap)}
|
||||
while True:
|
||||
if hasattr(unwrap, "__wrapped__"):
|
||||
unwrap = unwrap.__wrapped__
|
||||
candidate = unwrap.__wrapped__
|
||||
if id(candidate) in _seen_ids:
|
||||
break
|
||||
_seen_ids.add(id(candidate))
|
||||
unwrap = candidate
|
||||
continue
|
||||
if functools := sys.modules.get("functools"):
|
||||
if isinstance(unwrap, functools.partial):
|
||||
unwrap = unwrap.func
|
||||
candidate = unwrap.func
|
||||
if id(candidate) in _seen_ids:
|
||||
break
|
||||
_seen_ids.add(id(candidate))
|
||||
unwrap = candidate
|
||||
continue
|
||||
break
|
||||
if hasattr(unwrap, "__globals__"):
|
||||
@@ -1150,3 +1184,14 @@ def _get_dunder_annotations(obj):
|
||||
if not isinstance(ann, dict):
|
||||
raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None")
|
||||
return ann
|
||||
|
||||
|
||||
class _ExtraNameFixer(ast.NodeTransformer):
|
||||
"""Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation"""
|
||||
def __init__(self, extra_names):
|
||||
self.extra_names = extra_names
|
||||
|
||||
def visit_Name(self, node: ast.Name):
|
||||
if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel:
|
||||
node = ast.Name(id=type_repr(new_name))
|
||||
return node
|
||||
|
||||
12
Lib/argparse.py
vendored
12
Lib/argparse.py
vendored
@@ -149,6 +149,10 @@ def _copy_items(items):
|
||||
return copy.copy(items)
|
||||
|
||||
|
||||
def _identity(value):
|
||||
return value
|
||||
|
||||
|
||||
# ===============
|
||||
# Formatting Help
|
||||
# ===============
|
||||
@@ -200,7 +204,7 @@ class HelpFormatter(object):
|
||||
self._decolor = decolor
|
||||
else:
|
||||
self._theme = get_theme(force_no_color=True).argparse
|
||||
self._decolor = lambda text: text
|
||||
self._decolor = _identity
|
||||
|
||||
# ===============================
|
||||
# Section and indentation methods
|
||||
@@ -1903,9 +1907,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
self._subparsers = None
|
||||
|
||||
# register types
|
||||
def identity(string):
|
||||
return string
|
||||
self.register('type', None, identity)
|
||||
self.register('type', None, _identity)
|
||||
|
||||
# add help argument if necessary
|
||||
# (using explicit default to override global argument_default)
|
||||
@@ -2676,7 +2678,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
|
||||
|
||||
if value not in choices:
|
||||
args = {'value': str(value),
|
||||
'choices': ', '.join(map(str, action.choices))}
|
||||
'choices': ', '.join(repr(str(choice)) for choice in action.choices)}
|
||||
msg = _('invalid choice: %(value)r (choose from %(choices)s)')
|
||||
|
||||
if self.suggest_on_error and isinstance(value, str):
|
||||
|
||||
30
Lib/asyncio/__main__.py
vendored
30
Lib/asyncio/__main__.py
vendored
@@ -86,22 +86,27 @@ class REPLThread(threading.Thread):
|
||||
global return_code
|
||||
|
||||
try:
|
||||
banner = (
|
||||
f'asyncio REPL {sys.version} on {sys.platform}\n'
|
||||
f'Use "await" directly instead of "asyncio.run()".\n'
|
||||
f'Type "help", "copyright", "credits" or "license" '
|
||||
f'for more information.\n'
|
||||
)
|
||||
if not sys.flags.quiet:
|
||||
banner = (
|
||||
f'asyncio REPL {sys.version} on {sys.platform}\n'
|
||||
f'Use "await" directly instead of "asyncio.run()".\n'
|
||||
f'Type "help", "copyright", "credits" or "license" '
|
||||
f'for more information.\n'
|
||||
)
|
||||
|
||||
console.write(banner)
|
||||
console.write(banner)
|
||||
|
||||
if startup_path := os.getenv("PYTHONSTARTUP"):
|
||||
if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")):
|
||||
sys.audit("cpython.run_startup", startup_path)
|
||||
|
||||
import tokenize
|
||||
with tokenize.open(startup_path) as f:
|
||||
startup_code = compile(f.read(), startup_path, "exec")
|
||||
try:
|
||||
import tokenize
|
||||
with tokenize.open(startup_path) as f:
|
||||
startup_code = compile(f.read(), startup_path, "exec")
|
||||
exec(startup_code, console.locals)
|
||||
except SystemExit:
|
||||
raise
|
||||
except BaseException:
|
||||
console.showtraceback()
|
||||
|
||||
ps1 = getattr(sys, "ps1", ">>> ")
|
||||
if CAN_USE_PYREPL:
|
||||
@@ -236,4 +241,5 @@ if __name__ == '__main__':
|
||||
break
|
||||
|
||||
console.write('exiting asyncio REPL...\n')
|
||||
loop.close()
|
||||
sys.exit(return_code)
|
||||
|
||||
11
Lib/asyncio/base_events.py
vendored
11
Lib/asyncio/base_events.py
vendored
@@ -1345,6 +1345,17 @@ class BaseEventLoop(events.AbstractEventLoop):
|
||||
# have a chance to get called before "ssl_protocol.connection_made()".
|
||||
transport.pause_reading()
|
||||
|
||||
# gh-142352: move buffered StreamReader data to SSLProtocol
|
||||
if server_side:
|
||||
from .streams import StreamReaderProtocol
|
||||
if isinstance(protocol, StreamReaderProtocol):
|
||||
stream_reader = getattr(protocol, '_stream_reader', None)
|
||||
if stream_reader is not None:
|
||||
buffer = stream_reader._buffer
|
||||
if buffer:
|
||||
ssl_protocol._incoming.write(buffer)
|
||||
buffer.clear()
|
||||
|
||||
transport.set_protocol(ssl_protocol)
|
||||
conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
|
||||
resume_cb = self.call_soon(transport.resume_reading)
|
||||
|
||||
4
Lib/asyncio/base_subprocess.py
vendored
4
Lib/asyncio/base_subprocess.py
vendored
@@ -265,7 +265,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
# to avoid hanging forever in self._wait as otherwise _exit_waiters
|
||||
# would never be woken up, we wake them up here.
|
||||
for waiter in self._exit_waiters:
|
||||
if not waiter.cancelled():
|
||||
if not waiter.done():
|
||||
waiter.set_result(self._returncode)
|
||||
if all(p is not None and p.disconnected
|
||||
for p in self._pipes.values()):
|
||||
@@ -278,7 +278,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
|
||||
finally:
|
||||
# wake up futures waiting for wait()
|
||||
for waiter in self._exit_waiters:
|
||||
if not waiter.cancelled():
|
||||
if not waiter.done():
|
||||
waiter.set_result(self._returncode)
|
||||
self._exit_waiters = None
|
||||
self._loop = None
|
||||
|
||||
4
Lib/asyncio/futures.py
vendored
4
Lib/asyncio/futures.py
vendored
@@ -392,7 +392,7 @@ def _chain_future(source, destination):
|
||||
|
||||
def _call_check_cancel(destination):
|
||||
if destination.cancelled():
|
||||
if source_loop is None or source_loop is dest_loop:
|
||||
if source_loop is None or source_loop is events._get_running_loop():
|
||||
source.cancel()
|
||||
else:
|
||||
source_loop.call_soon_threadsafe(source.cancel)
|
||||
@@ -401,7 +401,7 @@ def _chain_future(source, destination):
|
||||
if (destination.cancelled() and
|
||||
dest_loop is not None and dest_loop.is_closed()):
|
||||
return
|
||||
if dest_loop is None or dest_loop is source_loop:
|
||||
if dest_loop is None or dest_loop is events._get_running_loop():
|
||||
_set_state(destination, source)
|
||||
else:
|
||||
if dest_loop.is_closed():
|
||||
|
||||
2
Lib/asyncio/queues.py
vendored
2
Lib/asyncio/queues.py
vendored
@@ -37,7 +37,7 @@ class Queue(mixins._LoopBoundMixin):
|
||||
is an integer greater than 0, then "await put()" will block when the
|
||||
queue reaches maxsize, until an item is removed by get().
|
||||
|
||||
Unlike the standard library Queue, you can reliably know this Queue's size
|
||||
Unlike queue.Queue, you can reliably know this Queue's size
|
||||
with qsize(), since your single-threaded asyncio application won't be
|
||||
interrupted between calling qsize() and doing an operation on the Queue.
|
||||
"""
|
||||
|
||||
26
Lib/asyncio/windows_utils.py
vendored
26
Lib/asyncio/windows_utils.py
vendored
@@ -10,7 +10,6 @@ import itertools
|
||||
import msvcrt
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import warnings
|
||||
|
||||
|
||||
@@ -24,6 +23,7 @@ BUFSIZE = 8192
|
||||
PIPE = subprocess.PIPE
|
||||
STDOUT = subprocess.STDOUT
|
||||
_mmap_counter = itertools.count()
|
||||
_MAX_PIPE_ATTEMPTS = 20
|
||||
|
||||
|
||||
# Replacement for os.pipe() using handles instead of fds
|
||||
@@ -31,10 +31,6 @@ _mmap_counter = itertools.count()
|
||||
|
||||
def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
|
||||
"""Like os.pipe() but with overlapped support and using handles not fds."""
|
||||
address = tempfile.mktemp(
|
||||
prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format(
|
||||
os.getpid(), next(_mmap_counter)))
|
||||
|
||||
if duplex:
|
||||
openmode = _winapi.PIPE_ACCESS_DUPLEX
|
||||
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
|
||||
@@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
|
||||
|
||||
h1 = h2 = None
|
||||
try:
|
||||
h1 = _winapi.CreateNamedPipe(
|
||||
address, openmode, _winapi.PIPE_WAIT,
|
||||
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
|
||||
for attempts in itertools.count():
|
||||
address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format(
|
||||
os.getpid(), next(_mmap_counter), os.urandom(8).hex())
|
||||
try:
|
||||
h1 = _winapi.CreateNamedPipe(
|
||||
address, openmode, _winapi.PIPE_WAIT,
|
||||
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
|
||||
break
|
||||
except OSError as e:
|
||||
if attempts >= _MAX_PIPE_ATTEMPTS:
|
||||
raise
|
||||
if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
|
||||
_winapi.ERROR_ACCESS_DENIED):
|
||||
raise
|
||||
|
||||
h2 = _winapi.CreateFile(
|
||||
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
|
||||
@@ -104,8 +111,9 @@ class PipeHandle:
|
||||
|
||||
def close(self, *, CloseHandle=_winapi.CloseHandle):
|
||||
if self._handle is not None:
|
||||
CloseHandle(self._handle)
|
||||
handle = self._handle
|
||||
self._handle = None
|
||||
CloseHandle(handle)
|
||||
|
||||
def __del__(self, _warn=warnings.warn):
|
||||
if self._handle is not None:
|
||||
|
||||
4
Lib/collections/_defaultdict.py
vendored
4
Lib/collections/_defaultdict.py
vendored
@@ -17,6 +17,10 @@ class defaultdict(dict):
|
||||
val = self.default_factory()
|
||||
else:
|
||||
raise KeyError(key)
|
||||
# CPython parity: a recursive __missing__ via factory() may have
|
||||
# already populated key; preserve that value instead of overwriting.
|
||||
if key in self:
|
||||
return self[key]
|
||||
self[key] = val
|
||||
return val
|
||||
|
||||
|
||||
17
Lib/configparser.py
vendored
17
Lib/configparser.py
vendored
@@ -315,12 +315,15 @@ class ParsingError(Error):
|
||||
|
||||
def append(self, lineno, line):
|
||||
self.errors.append((lineno, line))
|
||||
self.message += '\n\t[line %2d]: %s' % (lineno, repr(line))
|
||||
self.message += f'\n\t[line {lineno:2d}]: {line!r}'
|
||||
|
||||
def combine(self, others):
|
||||
messages = [self.message]
|
||||
for other in others:
|
||||
for error in other.errors:
|
||||
self.append(*error)
|
||||
for lineno, line in other.errors:
|
||||
self.errors.append((lineno, line))
|
||||
messages.append(f'\n\t[line {lineno:2d}]: {line!r}')
|
||||
self.message = "".join(messages)
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
@@ -613,7 +616,9 @@ class RawConfigParser(MutableMapping):
|
||||
\] # ]
|
||||
"""
|
||||
_OPT_TMPL = r"""
|
||||
(?P<option>.*?) # very permissive!
|
||||
(?P<option> # very permissive!
|
||||
(?:(?!{delim})\S)* # non-delimiter non-whitespace
|
||||
(?:\s+(?:(?!{delim})\S)+)*) # optionally more words
|
||||
\s*(?P<vi>{delim})\s* # any number of space/tab,
|
||||
# followed by any of the
|
||||
# allowed delimiters,
|
||||
@@ -621,7 +626,9 @@ class RawConfigParser(MutableMapping):
|
||||
(?P<value>.*)$ # everything up to eol
|
||||
"""
|
||||
_OPT_NV_TMPL = r"""
|
||||
(?P<option>.*?) # very permissive!
|
||||
(?P<option> # very permissive!
|
||||
(?:(?!{delim})\S)* # non-delimiter non-whitespace
|
||||
(?:\s+(?:(?!{delim})\S)+)*) # optionally more words
|
||||
\s*(?: # any number of space/tab,
|
||||
(?P<vi>{delim})\s* # optionally followed by
|
||||
# any of the allowed
|
||||
|
||||
2
Lib/ctypes/__init__.py
vendored
2
Lib/ctypes/__init__.py
vendored
@@ -470,6 +470,8 @@ class CDLL(object):
|
||||
if name and name.endswith(")") and ".a(" in name:
|
||||
mode |= _os.RTLD_MEMBER | _os.RTLD_NOW
|
||||
self._name = name
|
||||
if handle is not None:
|
||||
return handle
|
||||
return _dlopen(name, mode)
|
||||
|
||||
def __repr__(self):
|
||||
|
||||
26
Lib/ctypes/util.py
vendored
26
Lib/ctypes/util.py
vendored
@@ -85,15 +85,10 @@ if os.name == "nt":
|
||||
wintypes.DWORD,
|
||||
)
|
||||
|
||||
_psapi = ctypes.WinDLL('psapi', use_last_error=True)
|
||||
_enum_process_modules = _psapi["EnumProcessModules"]
|
||||
_enum_process_modules.restype = wintypes.BOOL
|
||||
_enum_process_modules.argtypes = (
|
||||
wintypes.HANDLE,
|
||||
ctypes.POINTER(wintypes.HMODULE),
|
||||
wintypes.DWORD,
|
||||
wintypes.LPDWORD,
|
||||
)
|
||||
# gh-145307: We defer loading psapi.dll until _get_module_handles is called.
|
||||
# Loading additional DLLs at startup for functionality that may never be
|
||||
# used is wasteful.
|
||||
_enum_process_modules = None
|
||||
|
||||
def _get_module_filename(module: wintypes.HMODULE):
|
||||
name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS
|
||||
@@ -101,8 +96,19 @@ if os.name == "nt":
|
||||
return name.value
|
||||
return None
|
||||
|
||||
|
||||
def _get_module_handles():
|
||||
global _enum_process_modules
|
||||
if _enum_process_modules is None:
|
||||
_psapi = ctypes.WinDLL('psapi', use_last_error=True)
|
||||
_enum_process_modules = _psapi["EnumProcessModules"]
|
||||
_enum_process_modules.restype = wintypes.BOOL
|
||||
_enum_process_modules.argtypes = (
|
||||
wintypes.HANDLE,
|
||||
ctypes.POINTER(wintypes.HMODULE),
|
||||
wintypes.DWORD,
|
||||
wintypes.LPDWORD,
|
||||
)
|
||||
|
||||
process = _get_current_process()
|
||||
space_needed = wintypes.DWORD()
|
||||
n = 1024
|
||||
|
||||
26
Lib/dataclasses.py
vendored
26
Lib/dataclasses.py
vendored
@@ -725,10 +725,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init,
|
||||
annotation_fields=annotation_fields)
|
||||
|
||||
|
||||
def _frozen_get_del_attr(cls, fields, func_builder):
|
||||
locals = {'cls': cls,
|
||||
def _frozen_set_del_attr(cls, fields, func_builder):
|
||||
locals = {'__class__': cls,
|
||||
'FrozenInstanceError': FrozenInstanceError}
|
||||
condition = 'type(self) is cls'
|
||||
condition = 'type(self) is __class__'
|
||||
if fields:
|
||||
condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}'
|
||||
|
||||
@@ -736,14 +736,14 @@ def _frozen_get_del_attr(cls, fields, func_builder):
|
||||
('self', 'name', 'value'),
|
||||
(f' if {condition}:',
|
||||
' raise FrozenInstanceError(f"cannot assign to field {name!r}")',
|
||||
f' super(cls, self).__setattr__(name, value)'),
|
||||
f' super(__class__, self).__setattr__(name, value)'),
|
||||
locals=locals,
|
||||
overwrite_error=True)
|
||||
func_builder.add_fn('__delattr__',
|
||||
('self', 'name'),
|
||||
(f' if {condition}:',
|
||||
' raise FrozenInstanceError(f"cannot delete field {name!r}")',
|
||||
f' super(cls, self).__delattr__(name)'),
|
||||
f' super(__class__, self).__delattr__(name)'),
|
||||
locals=locals,
|
||||
overwrite_error=True)
|
||||
|
||||
@@ -1199,7 +1199,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
|
||||
overwrite_error='Consider using functools.total_ordering')
|
||||
|
||||
if frozen:
|
||||
_frozen_get_del_attr(cls, field_list, func_builder)
|
||||
_frozen_set_del_attr(cls, field_list, func_builder)
|
||||
|
||||
# Decide if/how we're going to create a hash function.
|
||||
hash_action = _hash_action[bool(unsafe_hash),
|
||||
@@ -1292,10 +1292,18 @@ def _update_func_cell_for__class__(f, oldcls, newcls):
|
||||
# This function doesn't reference __class__, so nothing to do.
|
||||
return False
|
||||
# Fix the cell to point to the new class, if it's already pointing
|
||||
# at the old class. I'm not convinced that the "is oldcls" test
|
||||
# is needed, but other than performance can't hurt.
|
||||
# at the old class.
|
||||
closure = f.__closure__[idx]
|
||||
if closure.cell_contents is oldcls:
|
||||
|
||||
try:
|
||||
contents = closure.cell_contents
|
||||
except ValueError:
|
||||
# Cell is empty
|
||||
return False
|
||||
|
||||
# This check makes it so we avoid updating an incorrect cell if the
|
||||
# class body contains a function that was defined in a different class.
|
||||
if contents is oldcls:
|
||||
closure.cell_contents = newcls
|
||||
return True
|
||||
return False
|
||||
|
||||
2
Lib/email/__init__.py
vendored
2
Lib/email/__init__.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2007 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/_encoded_words.py
vendored
2
Lib/email/_encoded_words.py
vendored
@@ -219,7 +219,7 @@ def encode(string, charset='utf-8', encoding=None, lang=''):
|
||||
|
||||
"""
|
||||
if charset == 'unknown-8bit':
|
||||
bstring = string.encode('ascii', 'surrogateescape')
|
||||
bstring = string.encode('utf-8', 'surrogateescape')
|
||||
else:
|
||||
bstring = string.encode(charset)
|
||||
if encoding is None:
|
||||
|
||||
138
Lib/email/_header_value_parser.py
vendored
138
Lib/email/_header_value_parser.py
vendored
@@ -80,7 +80,8 @@ from email import utils
|
||||
# Useful constants and functions
|
||||
#
|
||||
|
||||
WSP = set(' \t')
|
||||
_WSP = ' \t'
|
||||
WSP = set(_WSP)
|
||||
CFWS_LEADER = WSP | set('(')
|
||||
SPECIALS = set(r'()<>@,:;.\"[]')
|
||||
ATOM_ENDS = SPECIALS | WSP
|
||||
@@ -101,6 +102,12 @@ def make_quoted_pairs(value):
|
||||
return str(value).replace('\\', '\\\\').replace('"', '\\"')
|
||||
|
||||
|
||||
def make_parenthesis_pairs(value):
|
||||
"""Escape parenthesis and backslash for use within a comment."""
|
||||
return str(value).replace('\\', '\\\\') \
|
||||
.replace('(', '\\(').replace(')', '\\)')
|
||||
|
||||
|
||||
def quote_string(value):
|
||||
escaped = make_quoted_pairs(value)
|
||||
return f'"{escaped}"'
|
||||
@@ -632,11 +639,11 @@ class LocalPart(TokenList):
|
||||
for tok in self[0] + [DOT]:
|
||||
if tok.token_type == 'cfws':
|
||||
continue
|
||||
if (last_is_tl and tok.token_type == 'dot' and
|
||||
if (last_is_tl and tok.token_type == 'dot' and last and
|
||||
last[-1].token_type == 'cfws'):
|
||||
res[-1] = TokenList(last[:-1])
|
||||
is_tl = isinstance(tok, TokenList)
|
||||
if (is_tl and last.token_type == 'dot' and
|
||||
if (is_tl and last.token_type == 'dot' and tok and
|
||||
tok[0].token_type == 'cfws'):
|
||||
res.append(TokenList(tok[1:]))
|
||||
else:
|
||||
@@ -874,6 +881,12 @@ class MessageID(MsgID):
|
||||
class InvalidMessageID(MessageID):
|
||||
token_type = 'invalid-message-id'
|
||||
|
||||
class MessageIDList(TokenList):
|
||||
token_type = 'message-id-list'
|
||||
|
||||
@property
|
||||
def message_ids(self):
|
||||
return [x for x in self if x.token_type=='msg-id']
|
||||
|
||||
class Header(TokenList):
|
||||
token_type = 'header'
|
||||
@@ -933,7 +946,7 @@ class WhiteSpaceTerminal(Terminal):
|
||||
return ' '
|
||||
|
||||
def startswith_fws(self):
|
||||
return True
|
||||
return self and self[0] in WSP
|
||||
|
||||
|
||||
class ValueTerminal(Terminal):
|
||||
@@ -1232,8 +1245,7 @@ def get_bare_quoted_string(value):
|
||||
bare_quoted_string = BareQuotedString()
|
||||
value = value[1:]
|
||||
if value and value[0] == '"':
|
||||
token, value = get_qcontent(value)
|
||||
bare_quoted_string.append(token)
|
||||
return bare_quoted_string, value[1:]
|
||||
while value and value[0] != '"':
|
||||
if value[0] in WSP:
|
||||
token, value = get_fws(value)
|
||||
@@ -2046,12 +2058,10 @@ def get_address_list(value):
|
||||
address_list.defects.append(errors.InvalidHeaderDefect(
|
||||
"invalid address in address-list"))
|
||||
if value and value[0] != ',':
|
||||
# Crap after address; treat it as an invalid mailbox.
|
||||
# The mailbox info will still be available.
|
||||
mailbox = address_list[-1][0]
|
||||
mailbox.token_type = 'invalid-mailbox'
|
||||
# Crap after address: add it to the address list
|
||||
# as an invalid mailbox
|
||||
token, value = get_invalid_mailbox(value, ',')
|
||||
mailbox.extend(token)
|
||||
address_list.append(Address([token]))
|
||||
address_list.defects.append(errors.InvalidHeaderDefect(
|
||||
"invalid address in address-list"))
|
||||
if value: # Must be a , at this point.
|
||||
@@ -2171,6 +2181,32 @@ def parse_message_id(value):
|
||||
|
||||
return message_id
|
||||
|
||||
def parse_message_ids(value):
|
||||
"""in-reply-to = "In-Reply-To:" 1*msg-id CRLF
|
||||
references = "References:" 1*msg-id CRLF
|
||||
"""
|
||||
message_id_list = MessageIDList()
|
||||
while value:
|
||||
if value[0] == ',':
|
||||
# message id list separated with commas - this is invalid,
|
||||
# but happens rather frequently in the wild
|
||||
message_id_list.defects.append(
|
||||
errors.InvalidHeaderDefect("comma in msg-id list"))
|
||||
message_id_list.append(
|
||||
WhiteSpaceTerminal(' ', 'invalid-comma-replacement'))
|
||||
value = value[1:]
|
||||
continue
|
||||
try:
|
||||
token, value = get_msg_id(value)
|
||||
message_id_list.append(token)
|
||||
except errors.HeaderParseError as ex:
|
||||
token = get_unstructured(value)
|
||||
message_id_list.append(InvalidMessageID(token))
|
||||
message_id_list.defects.append(
|
||||
errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex)))
|
||||
break
|
||||
return message_id_list
|
||||
|
||||
#
|
||||
# XXX: As I begin to add additional header parsers, I'm realizing we probably
|
||||
# have two level of parser routines: the get_XXX methods that get a token in
|
||||
@@ -2788,8 +2824,12 @@ def _steal_trailing_WSP_if_exists(lines):
|
||||
if lines and lines[-1] and lines[-1][-1] in WSP:
|
||||
wsp = lines[-1][-1]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
# gh-142006: if the line is now empty, remove it entirely.
|
||||
if not lines[-1]:
|
||||
lines.pop()
|
||||
return wsp
|
||||
|
||||
|
||||
def _refold_parse_tree(parse_tree, *, policy):
|
||||
"""Return string of contents of parse_tree folded according to RFC rules.
|
||||
|
||||
@@ -2798,11 +2838,9 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
maxlen = policy.max_line_length or sys.maxsize
|
||||
encoding = 'utf-8' if policy.utf8 else 'us-ascii'
|
||||
lines = [''] # Folded lines to be output
|
||||
leading_whitespace = '' # When we have whitespace between two encoded
|
||||
# words, we may need to encode the whitespace
|
||||
# at the beginning of the second word.
|
||||
last_ew = None # Points to the last encoded character if there's an ew on
|
||||
# the line
|
||||
last_word_is_ew = False
|
||||
last_ew = None # if there is an encoded word in the last line of lines,
|
||||
# points to the encoded word's first character
|
||||
last_charset = None
|
||||
wrap_as_ew_blocked = 0
|
||||
want_encoding = False # This is set to True if we need to encode this part
|
||||
@@ -2837,6 +2875,7 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
if part.token_type == 'mime-parameters':
|
||||
# Mime parameter folding (using RFC2231) is extra special.
|
||||
_fold_mime_parameters(part, lines, maxlen, encoding)
|
||||
last_word_is_ew = False
|
||||
continue
|
||||
|
||||
if want_encoding and not wrap_as_ew_blocked:
|
||||
@@ -2853,6 +2892,7 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
# XXX what if encoded_part has no leading FWS?
|
||||
lines.append(newline)
|
||||
lines[-1] += encoded_part
|
||||
last_word_is_ew = False
|
||||
continue
|
||||
# Either this is not a major syntactic break, so we don't
|
||||
# want it on a line by itself even if it fits, or it
|
||||
@@ -2871,11 +2911,16 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
(last_charset == 'unknown-8bit' or
|
||||
last_charset == 'utf-8' and charset != 'us-ascii')):
|
||||
last_ew = None
|
||||
last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew,
|
||||
part.ew_combine_allowed, charset, leading_whitespace)
|
||||
# This whitespace has been added to the lines in _fold_as_ew()
|
||||
# so clear it now.
|
||||
leading_whitespace = ''
|
||||
last_ew = _fold_as_ew(
|
||||
tstr,
|
||||
lines,
|
||||
maxlen,
|
||||
last_ew,
|
||||
part.ew_combine_allowed,
|
||||
charset,
|
||||
last_word_is_ew,
|
||||
)
|
||||
last_word_is_ew = True
|
||||
last_charset = charset
|
||||
want_encoding = False
|
||||
continue
|
||||
@@ -2888,28 +2933,19 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
|
||||
if len(tstr) <= maxlen - len(lines[-1]):
|
||||
lines[-1] += tstr
|
||||
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
|
||||
continue
|
||||
|
||||
# This part is too long to fit. The RFC wants us to break at
|
||||
# "major syntactic breaks", so unless we don't consider this
|
||||
# to be one, check if it will fit on the next line by itself.
|
||||
leading_whitespace = ''
|
||||
if (part.syntactic_break and
|
||||
len(tstr) + 1 <= maxlen):
|
||||
newline = _steal_trailing_WSP_if_exists(lines)
|
||||
if newline or part.startswith_fws():
|
||||
# We're going to fold the data onto a new line here. Due to
|
||||
# the way encoded strings handle continuation lines, we need to
|
||||
# be prepared to encode any whitespace if the next line turns
|
||||
# out to start with an encoded word.
|
||||
lines.append(newline + tstr)
|
||||
|
||||
whitespace_accumulator = []
|
||||
for char in lines[-1]:
|
||||
if char not in WSP:
|
||||
break
|
||||
whitespace_accumulator.append(char)
|
||||
leading_whitespace = ''.join(whitespace_accumulator)
|
||||
last_word_is_ew = (last_word_is_ew
|
||||
and not bool(lines[-1].strip(_WSP)))
|
||||
last_ew = None
|
||||
continue
|
||||
if not hasattr(part, 'encode'):
|
||||
@@ -2924,6 +2960,13 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
[ValueTerminal(make_quoted_pairs(p), 'ptext')
|
||||
for p in newparts] +
|
||||
[ValueTerminal('"', 'ptext')])
|
||||
if part.token_type == 'comment':
|
||||
newparts = (
|
||||
[ValueTerminal('(', 'ptext')] +
|
||||
[ValueTerminal(make_parenthesis_pairs(p), 'ptext')
|
||||
if p.token_type == 'ptext' else p
|
||||
for p in newparts] +
|
||||
[ValueTerminal(')', 'ptext')])
|
||||
if not part.as_ew_allowed:
|
||||
wrap_as_ew_blocked += 1
|
||||
newparts.append(end_ew_not_allowed)
|
||||
@@ -2942,10 +2985,11 @@ def _refold_parse_tree(parse_tree, *, policy):
|
||||
else:
|
||||
# We can't fold it onto the next line either...
|
||||
lines[-1] += tstr
|
||||
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
|
||||
|
||||
return policy.linesep.join(lines) + policy.linesep
|
||||
|
||||
def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace):
|
||||
def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew):
|
||||
"""Fold string to_encode into lines as encoded word, combining if allowed.
|
||||
Return the new value for last_ew, or None if ew_combine_allowed is False.
|
||||
|
||||
@@ -2960,6 +3004,16 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
|
||||
to_encode = str(
|
||||
get_unstructured(lines[-1][last_ew:] + to_encode))
|
||||
lines[-1] = lines[-1][:last_ew]
|
||||
elif last_word_is_ew:
|
||||
# If we are following up an encoded word with another encoded word,
|
||||
# any white space between the two will be ignored when decoded.
|
||||
# Therefore, we encode all to-be-displayed whitespace in the second
|
||||
# encoded word.
|
||||
len_without_wsp = len(lines[-1].rstrip(_WSP))
|
||||
leading_whitespace = lines[-1][len_without_wsp:]
|
||||
lines[-1] = (lines[-1][:len_without_wsp]
|
||||
+ (' ' if leading_whitespace else ''))
|
||||
to_encode = leading_whitespace + to_encode
|
||||
elif to_encode[0] in WSP:
|
||||
# We're joining this to non-encoded text, so don't encode
|
||||
# the leading blank.
|
||||
@@ -2988,20 +3042,13 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
|
||||
|
||||
while to_encode:
|
||||
remaining_space = maxlen - len(lines[-1])
|
||||
text_space = remaining_space - chrome_len - len(leading_whitespace)
|
||||
text_space = remaining_space - chrome_len
|
||||
if text_space <= 0:
|
||||
lines.append(' ')
|
||||
newline = _steal_trailing_WSP_if_exists(lines)
|
||||
lines.append(newline or ' ')
|
||||
new_last_ew = len(lines[-1])
|
||||
continue
|
||||
|
||||
# If we are at the start of a continuation line, prepend whitespace
|
||||
# (we only want to do this when the line starts with an encoded word
|
||||
# but if we're folding in this helper function, then we know that we
|
||||
# are going to be writing out an encoded word.)
|
||||
if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace:
|
||||
encoded_word = _ew.encode(leading_whitespace, charset=encode_as)
|
||||
lines[-1] += encoded_word
|
||||
leading_whitespace = ''
|
||||
|
||||
to_encode_word = to_encode[:text_space]
|
||||
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
|
||||
excess = len(encoded_word) - remaining_space
|
||||
@@ -3013,7 +3060,6 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
|
||||
excess = len(encoded_word) - remaining_space
|
||||
lines[-1] += encoded_word
|
||||
to_encode = to_encode[len(to_encode_word):]
|
||||
leading_whitespace = ''
|
||||
|
||||
if to_encode:
|
||||
lines.append(' ')
|
||||
|
||||
14
Lib/email/_parseaddr.py
vendored
14
Lib/email/_parseaddr.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2002-2007 Python Software Foundation
|
||||
# Copyright (C) 2002 Python Software Foundation
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
"""Email address parsing code.
|
||||
@@ -225,7 +225,7 @@ class AddrlistClass:
|
||||
def __init__(self, field):
|
||||
"""Initialize a new instance.
|
||||
|
||||
`field' is an unparsed address header field, containing
|
||||
'field' is an unparsed address header field, containing
|
||||
one or more addresses.
|
||||
"""
|
||||
self.specials = '()<>@,:;.\"[]'
|
||||
@@ -426,14 +426,14 @@ class AddrlistClass:
|
||||
def getdelimited(self, beginchar, endchars, allowcomments=True):
|
||||
"""Parse a header fragment delimited by special characters.
|
||||
|
||||
`beginchar' is the start character for the fragment.
|
||||
If self is not looking at an instance of `beginchar' then
|
||||
'beginchar' is the start character for the fragment.
|
||||
If self is not looking at an instance of 'beginchar' then
|
||||
getdelimited returns the empty string.
|
||||
|
||||
`endchars' is a sequence of allowable end-delimiting characters.
|
||||
'endchars' is a sequence of allowable end-delimiting characters.
|
||||
Parsing stops when one of these is encountered.
|
||||
|
||||
If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed
|
||||
If 'allowcomments' is non-zero, embedded RFC 2822 comments are allowed
|
||||
within the parsed fragment.
|
||||
"""
|
||||
if self.field[self.pos] != beginchar:
|
||||
@@ -477,7 +477,7 @@ class AddrlistClass:
|
||||
|
||||
Optional atomends specifies a different set of end token delimiters
|
||||
(the default is to use self.atomends). This is used e.g. in
|
||||
getphraselist() since phrase endings must not include the `.' (which
|
||||
getphraselist() since phrase endings must not include the '.' (which
|
||||
is legal in phrases)."""
|
||||
atomlist = ['']
|
||||
if atomends is None:
|
||||
|
||||
12
Lib/email/_policybase.py
vendored
12
Lib/email/_policybase.py
vendored
@@ -4,6 +4,7 @@ Allows fine grained feature control of how the package parses and emits data.
|
||||
"""
|
||||
|
||||
import abc
|
||||
import re
|
||||
from email import header
|
||||
from email import charset as _charset
|
||||
from email.utils import _has_surrogates
|
||||
@@ -14,6 +15,14 @@ __all__ = [
|
||||
'compat32',
|
||||
]
|
||||
|
||||
# validation regex from RFC 5322, equivalent to pattern re.compile("[!-9;-~]+$")
|
||||
valid_header_name_re = re.compile("[\041-\071\073-\176]+$")
|
||||
|
||||
def validate_header_name(name):
|
||||
# Validate header name according to RFC 5322
|
||||
if not valid_header_name_re.match(name):
|
||||
raise ValueError(
|
||||
f"Header field name contains invalid characters: {name!r}")
|
||||
|
||||
class _PolicyBase:
|
||||
|
||||
@@ -150,7 +159,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
|
||||
wrapping is done. Default is 78.
|
||||
|
||||
mangle_from_ -- a flag that, when True escapes From_ lines in the
|
||||
body of the message by putting a `>' in front of
|
||||
body of the message by putting a '>' in front of
|
||||
them. This is used when the message is being
|
||||
serialized by a generator. Default: False.
|
||||
|
||||
@@ -314,6 +323,7 @@ class Compat32(Policy):
|
||||
"""+
|
||||
The name and value are returned unmodified.
|
||||
"""
|
||||
validate_header_name(name)
|
||||
return (name, value)
|
||||
|
||||
def header_fetch_parse(self, name, value):
|
||||
|
||||
4
Lib/email/base64mime.py
vendored
4
Lib/email/base64mime.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2002-2007 Python Software Foundation
|
||||
# Copyright (C) 2002 Python Software Foundation
|
||||
# Author: Ben Gertzfield
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -15,7 +15,7 @@ This module provides an interface to encode and decode both headers and bodies
|
||||
with Base64 encoding.
|
||||
|
||||
RFC 2045 defines a method for including character set information in an
|
||||
`encoded-word' in a header. This method is commonly used for 8-bit real names
|
||||
'encoded-word' in a header. This method is commonly used for 8-bit real names
|
||||
in To:, From:, Cc:, etc. fields, as well as Subject: lines.
|
||||
|
||||
This module does not do the line wrapping or end-of-line character conversion
|
||||
|
||||
6
Lib/email/charset.py
vendored
6
Lib/email/charset.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2007 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Ben Gertzfield, Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -175,7 +175,7 @@ class Charset:
|
||||
module expose the following information about a character set:
|
||||
|
||||
input_charset: The initial character set specified. Common aliases
|
||||
are converted to their `official' email names (e.g. latin_1
|
||||
are converted to their 'official' email names (e.g. latin_1
|
||||
is converted to iso-8859-1). Defaults to 7-bit us-ascii.
|
||||
|
||||
header_encoding: If the character set must be encoded before it can be
|
||||
@@ -245,7 +245,7 @@ class Charset:
|
||||
def get_body_encoding(self):
|
||||
"""Return the content-transfer-encoding used for body encoding.
|
||||
|
||||
This is either the string `quoted-printable' or `base64' depending on
|
||||
This is either the string 'quoted-printable' or 'base64' depending on
|
||||
the encoding used, or it is a function in which case you should call
|
||||
the function with a single argument, the Message object being
|
||||
encoded. The function should then set the Content-Transfer-Encoding
|
||||
|
||||
2
Lib/email/encoders.py
vendored
2
Lib/email/encoders.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/errors.py
vendored
2
Lib/email/errors.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
11
Lib/email/feedparser.py
vendored
11
Lib/email/feedparser.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2004-2006 Python Software Foundation
|
||||
# Copyright (C) 2004 Python Software Foundation
|
||||
# Authors: Baxter, Wouters and Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -30,7 +30,7 @@ from io import StringIO
|
||||
|
||||
NLCRE = re.compile(r'\r\n|\r|\n')
|
||||
NLCRE_bol = re.compile(r'(\r\n|\r|\n)')
|
||||
NLCRE_eol = re.compile(r'(\r\n|\r|\n)\Z')
|
||||
NLCRE_eol = re.compile(r'(\r\n|\r|\n)\z')
|
||||
NLCRE_crack = re.compile(r'(\r\n|\r|\n)')
|
||||
# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character
|
||||
# except controls, SP, and ":".
|
||||
@@ -504,10 +504,9 @@ class FeedParser:
|
||||
self._input.unreadline(line)
|
||||
return
|
||||
else:
|
||||
# Weirdly placed unix-from line. Note this as a defect
|
||||
# and ignore it.
|
||||
# Weirdly placed unix-from line.
|
||||
defect = errors.MisplacedEnvelopeHeaderDefect(line)
|
||||
self._cur.defects.append(defect)
|
||||
self.policy.handle_defect(self._cur, defect)
|
||||
continue
|
||||
# Split the line on the colon separating field name from value.
|
||||
# There will always be a colon, because if there wasn't the part of
|
||||
@@ -519,7 +518,7 @@ class FeedParser:
|
||||
# message. Track the error but keep going.
|
||||
if i == 0:
|
||||
defect = errors.InvalidHeaderDefect("Missing header name.")
|
||||
self._cur.defects.append(defect)
|
||||
self.policy.handle_defect(self._cur, defect)
|
||||
continue
|
||||
|
||||
assert i>0, "_parse_headers fed line with no : and no leading WS"
|
||||
|
||||
24
Lib/email/generator.py
vendored
24
Lib/email/generator.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2010 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -22,6 +22,7 @@ NL = '\n' # XXX: no longer used by the code below.
|
||||
NLCRE = re.compile(r'\r\n|\r|\n')
|
||||
fcre = re.compile(r'^From ', re.MULTILINE)
|
||||
NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
|
||||
NEWLINE_WITHOUT_FWSP_BYTES = re.compile(br'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
|
||||
|
||||
|
||||
class Generator:
|
||||
@@ -43,7 +44,7 @@ class Generator:
|
||||
|
||||
Optional mangle_from_ is a flag that, when True (the default if policy
|
||||
is not set), escapes From_ lines in the body of the message by putting
|
||||
a `>' in front of them.
|
||||
a '>' in front of them.
|
||||
|
||||
Optional maxheaderlen specifies the longest length for a non-continued
|
||||
header. When a header line is longer (in characters, with tabs
|
||||
@@ -76,7 +77,7 @@ class Generator:
|
||||
|
||||
unixfrom is a flag that forces the printing of a Unix From_ delimiter
|
||||
before the first object in the message tree. If the original message
|
||||
has no From_ delimiter, a `standard' one is crafted. By default, this
|
||||
has no From_ delimiter, a 'standard' one is crafted. By default, this
|
||||
is False to inhibit the printing of any From_ delimiter.
|
||||
|
||||
Note that for subobjects, no From_ line is printed.
|
||||
@@ -227,7 +228,7 @@ class Generator:
|
||||
folded = self.policy.fold(h, v)
|
||||
if self.policy.verify_generated_headers:
|
||||
linesep = self.policy.linesep
|
||||
if not folded.endswith(self.policy.linesep):
|
||||
if not folded.endswith(linesep):
|
||||
raise HeaderWriteError(
|
||||
f'folded header does not end with {linesep!r}: {folded!r}')
|
||||
if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
|
||||
@@ -391,7 +392,7 @@ class Generator:
|
||||
b = boundary
|
||||
counter = 0
|
||||
while True:
|
||||
cre = cls._compile_re('^--' + re.escape(b) + '(--)?$', re.MULTILINE)
|
||||
cre = cls._compile_re('^--' + re.escape(b) + '(--)?\r?$', re.MULTILINE)
|
||||
if not cre.search(text):
|
||||
break
|
||||
b = boundary + '.' + str(counter)
|
||||
@@ -429,7 +430,16 @@ class BytesGenerator(Generator):
|
||||
# This is almost the same as the string version, except for handling
|
||||
# strings with 8bit bytes.
|
||||
for h, v in msg.raw_items():
|
||||
self._fp.write(self.policy.fold_binary(h, v))
|
||||
folded = self.policy.fold_binary(h, v)
|
||||
if self.policy.verify_generated_headers:
|
||||
linesep = self.policy.linesep.encode()
|
||||
if not folded.endswith(linesep):
|
||||
raise HeaderWriteError(
|
||||
f'folded header does not end with {linesep!r}: {folded!r}')
|
||||
if NEWLINE_WITHOUT_FWSP_BYTES.search(folded.removesuffix(linesep)):
|
||||
raise HeaderWriteError(
|
||||
f'folded header contains newline: {folded!r}')
|
||||
self._fp.write(folded)
|
||||
# A blank line always separates headers from body
|
||||
self.write(self._NL)
|
||||
|
||||
@@ -467,7 +477,7 @@ class DecodedGenerator(Generator):
|
||||
argument is allowed.
|
||||
|
||||
Walks through all subparts of a message. If the subpart is of main
|
||||
type `text', then it prints the decoded payload of the subpart.
|
||||
type 'text', then it prints the decoded payload of the subpart.
|
||||
|
||||
Otherwise, fmt is a format string that is used instead of the message
|
||||
payload. fmt is expanded with the following keywords (in
|
||||
|
||||
8
Lib/email/header.py
vendored
8
Lib/email/header.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2002-2007 Python Software Foundation
|
||||
# Copyright (C) 2002 Python Software Foundation
|
||||
# Author: Ben Gertzfield, Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -201,7 +201,7 @@ class Header:
|
||||
|
||||
The maximum line length can be specified explicitly via maxlinelen. For
|
||||
splitting the first line to a shorter value (to account for the field
|
||||
header which isn't included in s, e.g. `Subject') pass in the name of
|
||||
header which isn't included in s, e.g. 'Subject') pass in the name of
|
||||
the field in header_name. The default maxlinelen is 78 as recommended
|
||||
by RFC 2822.
|
||||
|
||||
@@ -285,7 +285,7 @@ class Header:
|
||||
output codec of the charset. If the string cannot be encoded to the
|
||||
output codec, a UnicodeError will be raised.
|
||||
|
||||
Optional `errors' is passed as the errors argument to the decode
|
||||
Optional 'errors' is passed as the errors argument to the decode
|
||||
call if s is a byte string.
|
||||
"""
|
||||
if charset is None:
|
||||
@@ -335,7 +335,7 @@ class Header:
|
||||
|
||||
Optional splitchars is a string containing characters which should be
|
||||
given extra weight by the splitting algorithm during normal header
|
||||
wrapping. This is in very rough support of RFC 2822's `higher level
|
||||
wrapping. This is in very rough support of RFC 2822's 'higher level
|
||||
syntactic breaks': split points preceded by a splitchar are preferred
|
||||
during line splitting, with the characters preferred in the order in
|
||||
which they appear in the string. Space and tab may be included in the
|
||||
|
||||
14
Lib/email/headerregistry.py
vendored
14
Lib/email/headerregistry.py
vendored
@@ -534,6 +534,18 @@ class MessageIDHeader:
|
||||
kwds['defects'].extend(parse_tree.all_defects)
|
||||
|
||||
|
||||
class ReferencesHeader:
|
||||
|
||||
max_count = 1
|
||||
value_parser = staticmethod(parser.parse_message_ids)
|
||||
|
||||
@classmethod
|
||||
def parse(cls, value, kwds):
|
||||
kwds['parse_tree'] = parse_tree = cls.value_parser(value)
|
||||
kwds['decoded'] = str(parse_tree)
|
||||
kwds['defects'].extend(parse_tree.all_defects)
|
||||
|
||||
|
||||
# The header factory #
|
||||
|
||||
_default_header_map = {
|
||||
@@ -557,6 +569,8 @@ _default_header_map = {
|
||||
'content-disposition': ContentDispositionHeader,
|
||||
'content-transfer-encoding': ContentTransferEncodingHeader,
|
||||
'message-id': MessageIDHeader,
|
||||
'in-reply-to': ReferencesHeader,
|
||||
'references': ReferencesHeader,
|
||||
}
|
||||
|
||||
class HeaderRegistry:
|
||||
|
||||
6
Lib/email/iterators.py
vendored
6
Lib/email/iterators.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -43,8 +43,8 @@ def body_line_iterator(msg, decode=False):
|
||||
def typed_subpart_iterator(msg, maintype='text', subtype=None):
|
||||
"""Iterate over the subparts with a given MIME type.
|
||||
|
||||
Use `maintype' as the main MIME type to match against; this defaults to
|
||||
"text". Optional `subtype' is the MIME subtype to match against; if
|
||||
Use 'maintype' as the main MIME type to match against; this defaults to
|
||||
"text". Optional 'subtype' is the MIME subtype to match against; if
|
||||
omitted, only the main type is matched.
|
||||
"""
|
||||
for subpart in msg.walk():
|
||||
|
||||
28
Lib/email/message.py
vendored
28
Lib/email/message.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2007 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -21,7 +21,7 @@ Charset = _charset.Charset
|
||||
|
||||
SEMISPACE = '; '
|
||||
|
||||
# Regular expression that matches `special' characters in parameters, the
|
||||
# Regular expression that matches 'special' characters in parameters, the
|
||||
# existence of which force quoting of the parameter value.
|
||||
tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]')
|
||||
|
||||
@@ -147,7 +147,7 @@ class Message:
|
||||
multipart or a message/rfc822), then the payload is a list of Message
|
||||
objects, otherwise it is a string.
|
||||
|
||||
Message objects implement part of the `mapping' interface, which assumes
|
||||
Message objects implement part of the 'mapping' interface, which assumes
|
||||
there is exactly one occurrence of the header per message. Some headers
|
||||
do in fact appear multiple times (e.g. Received) and for those headers,
|
||||
you must use the explicit API to set or get all the headers. Not all of
|
||||
@@ -609,7 +609,7 @@ class Message:
|
||||
"""Return the message's content type.
|
||||
|
||||
The returned string is coerced to lower case of the form
|
||||
`maintype/subtype'. If there was no Content-Type header in the
|
||||
'maintype/subtype'. If there was no Content-Type header in the
|
||||
message, the default type as given by get_default_type() will be
|
||||
returned. Since according to RFC 2045, messages always have a default
|
||||
type this will always return a value.
|
||||
@@ -632,7 +632,7 @@ class Message:
|
||||
def get_content_maintype(self):
|
||||
"""Return the message's main content type.
|
||||
|
||||
This is the `maintype' part of the string returned by
|
||||
This is the 'maintype' part of the string returned by
|
||||
get_content_type().
|
||||
"""
|
||||
ctype = self.get_content_type()
|
||||
@@ -641,14 +641,14 @@ class Message:
|
||||
def get_content_subtype(self):
|
||||
"""Returns the message's sub-content type.
|
||||
|
||||
This is the `subtype' part of the string returned by
|
||||
This is the 'subtype' part of the string returned by
|
||||
get_content_type().
|
||||
"""
|
||||
ctype = self.get_content_type()
|
||||
return ctype.split('/')[1]
|
||||
|
||||
def get_default_type(self):
|
||||
"""Return the `default' content type.
|
||||
"""Return the 'default' content type.
|
||||
|
||||
Most messages have a default content type of text/plain, except for
|
||||
messages that are subparts of multipart/digest containers. Such
|
||||
@@ -657,7 +657,7 @@ class Message:
|
||||
return self._default_type
|
||||
|
||||
def set_default_type(self, ctype):
|
||||
"""Set the `default' content type.
|
||||
"""Set the 'default' content type.
|
||||
|
||||
ctype should be either "text/plain" or "message/rfc822", although this
|
||||
is not enforced. The default content type is not stored in the
|
||||
@@ -690,8 +690,8 @@ class Message:
|
||||
"""Return the message's Content-Type parameters, as a list.
|
||||
|
||||
The elements of the returned list are 2-tuples of key/value pairs, as
|
||||
split on the `=' sign. The left hand side of the `=' is the key,
|
||||
while the right hand side is the value. If there is no `=' sign in
|
||||
split on the '=' sign. The left hand side of the '=' is the key,
|
||||
while the right hand side is the value. If there is no '=' sign in
|
||||
the parameter the value is the empty string. The value is as
|
||||
described in the get_param() method.
|
||||
|
||||
@@ -851,9 +851,9 @@ class Message:
|
||||
"""Return the filename associated with the payload if present.
|
||||
|
||||
The filename is extracted from the Content-Disposition header's
|
||||
`filename' parameter, and it is unquoted. If that header is missing
|
||||
the `filename' parameter, this method falls back to looking for the
|
||||
`name' parameter.
|
||||
'filename' parameter, and it is unquoted. If that header is missing
|
||||
the 'filename' parameter, this method falls back to looking for the
|
||||
'name' parameter.
|
||||
"""
|
||||
missing = object()
|
||||
filename = self.get_param('filename', missing, 'content-disposition')
|
||||
@@ -866,7 +866,7 @@ class Message:
|
||||
def get_boundary(self, failobj=None):
|
||||
"""Return the boundary associated with the payload if present.
|
||||
|
||||
The boundary is extracted from the Content-Type header's `boundary'
|
||||
The boundary is extracted from the Content-Type header's 'boundary'
|
||||
parameter, and it is unquoted.
|
||||
"""
|
||||
missing = object()
|
||||
|
||||
2
Lib/email/mime/application.py
vendored
2
Lib/email/mime/application.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Keith Dart
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/mime/audio.py
vendored
2
Lib/email/mime/audio.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2007 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Anthony Baxter
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/mime/base.py
vendored
2
Lib/email/mime/base.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/mime/image.py
vendored
2
Lib/email/mime/image.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/mime/message.py
vendored
2
Lib/email/mime/message.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
4
Lib/email/mime/multipart.py
vendored
4
Lib/email/mime/multipart.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2002-2006 Python Software Foundation
|
||||
# Copyright (C) 2002 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -21,7 +21,7 @@ class MIMEMultipart(MIMEBase):
|
||||
Content-Type and MIME-Version headers.
|
||||
|
||||
_subtype is the subtype of the multipart content type, defaulting to
|
||||
`mixed'.
|
||||
'mixed'.
|
||||
|
||||
boundary is the multipart boundary string. By default it is
|
||||
calculated as needed.
|
||||
|
||||
2
Lib/email/mime/nonmultipart.py
vendored
2
Lib/email/mime/nonmultipart.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2002-2006 Python Software Foundation
|
||||
# Copyright (C) 2002 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/mime/text.py
vendored
2
Lib/email/mime/text.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
2
Lib/email/parser.py
vendored
2
Lib/email/parser.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2007 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw, Thomas Wouters, Anthony Baxter
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
|
||||
9
Lib/email/policy.py
vendored
9
Lib/email/policy.py
vendored
@@ -4,7 +4,13 @@ code that adds all the email6 features.
|
||||
|
||||
import re
|
||||
import sys
|
||||
from email._policybase import Policy, Compat32, compat32, _extend_docstrings
|
||||
from email._policybase import (
|
||||
Compat32,
|
||||
Policy,
|
||||
_extend_docstrings,
|
||||
compat32,
|
||||
validate_header_name
|
||||
)
|
||||
from email.utils import _has_surrogates
|
||||
from email.headerregistry import HeaderRegistry as HeaderRegistry
|
||||
from email.contentmanager import raw_data_manager
|
||||
@@ -138,6 +144,7 @@ class EmailPolicy(Policy):
|
||||
CR or LF characters.
|
||||
|
||||
"""
|
||||
validate_header_name(name)
|
||||
if hasattr(value, 'name') and value.name.lower() == name.lower():
|
||||
return (name, value)
|
||||
if isinstance(value, str) and len(value.splitlines())>1:
|
||||
|
||||
12
Lib/email/quoprimime.py
vendored
12
Lib/email/quoprimime.py
vendored
@@ -1,11 +1,11 @@
|
||||
# Copyright (C) 2001-2006 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Ben Gertzfield
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
"""Quoted-printable content transfer encoding per RFCs 2045-2047.
|
||||
|
||||
This module handles the content transfer encoding method defined in RFC 2045
|
||||
to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to
|
||||
to encode US ASCII-like 8-bit data called 'quoted-printable'. It is used to
|
||||
safely encode text that is in a character set similar to the 7-bit US ASCII
|
||||
character set, but that includes some 8-bit characters that are normally not
|
||||
allowed in email bodies or headers.
|
||||
@@ -17,7 +17,7 @@ This module provides an interface to encode and decode both headers and bodies
|
||||
with quoted-printable encoding.
|
||||
|
||||
RFC 2045 defines a method for including character set information in an
|
||||
`encoded-word' in a header. This method is commonly used for 8-bit real names
|
||||
'encoded-word' in a header. This method is commonly used for 8-bit real names
|
||||
in To:/From:/Cc: etc. fields, as well as Subject: lines.
|
||||
|
||||
This module does not do the line wrapping or end-of-line character
|
||||
@@ -127,7 +127,7 @@ def quote(c):
|
||||
def header_encode(header_bytes, charset='iso-8859-1'):
|
||||
"""Encode a single header line with quoted-printable (like) encoding.
|
||||
|
||||
Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but
|
||||
Defined in RFC 2045, this 'Q' encoding is similar to quoted-printable, but
|
||||
used specifically for email header fields to allow charsets with mostly 7
|
||||
bit characters (and some 8 bit) to remain more or less readable in non-RFC
|
||||
2045 aware mail clients.
|
||||
@@ -272,7 +272,7 @@ def decode(encoded, eol=NL):
|
||||
decoded += eol
|
||||
# Special case if original string did not end with eol
|
||||
if encoded[-1] not in '\r\n' and decoded.endswith(eol):
|
||||
decoded = decoded[:-1]
|
||||
decoded = decoded[:-len(eol)]
|
||||
return decoded
|
||||
|
||||
|
||||
@@ -290,7 +290,7 @@ def _unquote_match(match):
|
||||
|
||||
# Header decoding is done a bit differently
|
||||
def header_decode(s):
|
||||
"""Decode a string encoded with RFC 2045 MIME header `Q' encoding.
|
||||
"""Decode a string encoded with RFC 2045 MIME header 'Q' encoding.
|
||||
|
||||
This function does not parse a full MIME header value encoded with
|
||||
quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use
|
||||
|
||||
12
Lib/email/utils.py
vendored
12
Lib/email/utils.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2001-2010 Python Software Foundation
|
||||
# Copyright (C) 2001 Python Software Foundation
|
||||
# Author: Barry Warsaw
|
||||
# Contact: email-sig@python.org
|
||||
|
||||
@@ -472,23 +472,15 @@ def collapse_rfc2231_value(value, errors='replace',
|
||||
# better than not having it.
|
||||
#
|
||||
|
||||
def localtime(dt=None, isdst=None):
|
||||
def localtime(dt=None):
|
||||
"""Return local time as an aware datetime object.
|
||||
|
||||
If called without arguments, return current time. Otherwise *dt*
|
||||
argument should be a datetime instance, and it is converted to the
|
||||
local time zone according to the system time zone database. If *dt* is
|
||||
naive (that is, dt.tzinfo is None), it is assumed to be in local time.
|
||||
The isdst parameter is ignored.
|
||||
|
||||
"""
|
||||
if isdst is not None:
|
||||
import warnings
|
||||
warnings._deprecated(
|
||||
"The 'isdst' parameter to 'localtime'",
|
||||
message='{name} is deprecated and slated for removal in Python {remove}',
|
||||
remove=(3, 14),
|
||||
)
|
||||
if dt is None:
|
||||
dt = datetime.datetime.now()
|
||||
return dt.astimezone()
|
||||
|
||||
5
Lib/encodings/__init__.py
vendored
5
Lib/encodings/__init__.py
vendored
@@ -33,6 +33,7 @@ import sys
|
||||
from . import aliases
|
||||
|
||||
_cache = {}
|
||||
_MAXCACHE = 500
|
||||
_unknown = '--unknown--'
|
||||
_import_tail = ['*']
|
||||
_aliases = aliases.aliases
|
||||
@@ -115,6 +116,8 @@ def search_function(encoding):
|
||||
|
||||
if mod is None:
|
||||
# Cache misses
|
||||
if len(_cache) >= _MAXCACHE:
|
||||
_cache.clear()
|
||||
_cache[encoding] = None
|
||||
return None
|
||||
|
||||
@@ -136,6 +139,8 @@ def search_function(encoding):
|
||||
entry = codecs.CodecInfo(*entry)
|
||||
|
||||
# Cache the codec registry entry
|
||||
if len(_cache) >= _MAXCACHE:
|
||||
_cache.clear()
|
||||
_cache[encoding] = entry
|
||||
|
||||
# Register its aliases (without overwriting previously registered
|
||||
|
||||
5
Lib/ensurepip/__init__.py
vendored
5
Lib/ensurepip/__init__.py
vendored
@@ -10,13 +10,14 @@ from shutil import copy2
|
||||
|
||||
|
||||
__all__ = ["version", "bootstrap"]
|
||||
_PIP_VERSION = "25.3"
|
||||
_PIP_VERSION = "26.1.1"
|
||||
|
||||
# Directory of system wheel packages. Some Linux distribution packaging
|
||||
# policies recommend against bundling dependencies. For example, Fedora
|
||||
# installs wheel packages in the /usr/share/python-wheels/ directory and don't
|
||||
# install the ensurepip._bundled package.
|
||||
if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None:
|
||||
_pkg_dir = sysconfig.get_config_var('WHEEL_PKG_DIR')
|
||||
if _pkg_dir:
|
||||
_WHEEL_PKG_DIR = Path(_pkg_dir).resolve()
|
||||
else:
|
||||
_WHEEL_PKG_DIR = None
|
||||
|
||||
Binary file not shown.
27
Lib/glob.py
vendored
27
Lib/glob.py
vendored
@@ -15,7 +15,7 @@ __all__ = ["glob", "iglob", "escape", "translate"]
|
||||
|
||||
def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
|
||||
include_hidden=False):
|
||||
"""Return a list of paths matching a pathname pattern.
|
||||
"""Return a list of paths matching a `pathname` pattern.
|
||||
|
||||
The pattern may contain simple shell-style wildcards a la
|
||||
fnmatch. Unlike fnmatch, filenames starting with a
|
||||
@@ -25,6 +25,15 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
|
||||
The order of the returned list is undefined. Sort it if you need a
|
||||
particular order.
|
||||
|
||||
If `root_dir` is not None, it should be a path-like object specifying the
|
||||
root directory for searching. It has the same effect as changing the
|
||||
current directory before calling it (without actually
|
||||
changing it). If pathname is relative, the result will contain
|
||||
paths relative to `root_dir`.
|
||||
|
||||
If `dir_fd` is not None, it should be a file descriptor referring to a
|
||||
directory, and paths will then be relative to that directory.
|
||||
|
||||
If `include_hidden` is true, the patterns '*', '?', '**' will match hidden
|
||||
directories.
|
||||
|
||||
@@ -36,7 +45,7 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
|
||||
|
||||
def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
|
||||
include_hidden=False):
|
||||
"""Return an iterator which yields the paths matching a pathname pattern.
|
||||
"""Return an iterator which yields the paths matching a `pathname` pattern.
|
||||
|
||||
The pattern may contain simple shell-style wildcards a la
|
||||
fnmatch. However, unlike fnmatch, filenames starting with a
|
||||
@@ -46,7 +55,19 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
|
||||
The order of the returned paths is undefined. Sort them if you need a
|
||||
particular order.
|
||||
|
||||
If recursive is true, the pattern '**' will match any files and
|
||||
If `root_dir` is not None, it should be a path-like object specifying
|
||||
the root directory for searching. It has the same effect as changing
|
||||
the current directory before calling it (without actually
|
||||
changing it). If pathname is relative, the result will contain
|
||||
paths relative to `root_dir`.
|
||||
|
||||
If `dir_fd` is not None, it should be a file descriptor referring to a
|
||||
directory, and paths will then be relative to that directory.
|
||||
|
||||
If `include_hidden` is true, the patterns '*', '?', '**' will match hidden
|
||||
directories.
|
||||
|
||||
If `recursive` is true, the pattern '**' will match any files and
|
||||
zero or more directories and subdirectories.
|
||||
"""
|
||||
sys.audit("glob.glob", pathname, recursive)
|
||||
|
||||
11
Lib/http/client.py
vendored
11
Lib/http/client.py
vendored
@@ -972,13 +972,22 @@ class HTTPConnection:
|
||||
return ip
|
||||
|
||||
def _tunnel(self):
|
||||
if _contains_disallowed_url_pchar_re.search(self._tunnel_host):
|
||||
raise ValueError('Tunnel host can\'t contain control characters %r'
|
||||
% (self._tunnel_host,))
|
||||
connect = b"CONNECT %s:%d %s\r\n" % (
|
||||
self._wrap_ipv6(self._tunnel_host.encode("idna")),
|
||||
self._tunnel_port,
|
||||
self._http_vsn_str.encode("ascii"))
|
||||
headers = [connect]
|
||||
for header, value in self._tunnel_headers.items():
|
||||
headers.append(f"{header}: {value}\r\n".encode("latin-1"))
|
||||
header_bytes = header.encode("latin-1")
|
||||
value_bytes = value.encode("latin-1")
|
||||
if not _is_legal_header_name(header_bytes):
|
||||
raise ValueError('Invalid header name %r' % (header_bytes,))
|
||||
if _is_illegal_header_value(value_bytes):
|
||||
raise ValueError('Invalid header value %r' % (value_bytes,))
|
||||
headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes))
|
||||
headers.append(b"\r\n")
|
||||
# Making a single send() call instead of one per line encourages
|
||||
# the host OS to use a more optimal packet size instead of
|
||||
|
||||
30
Lib/http/cookies.py
vendored
30
Lib/http/cookies.py
vendored
@@ -337,9 +337,16 @@ class Morsel(dict):
|
||||
key = key.lower()
|
||||
if key not in self._reserved:
|
||||
raise CookieError("Invalid attribute %r" % (key,))
|
||||
if _has_control_character(key, val):
|
||||
raise CookieError("Control characters are not allowed in "
|
||||
f"cookies {key!r} {val!r}")
|
||||
data[key] = val
|
||||
dict.update(self, data)
|
||||
|
||||
def __ior__(self, values):
|
||||
self.update(values)
|
||||
return self
|
||||
|
||||
def isReservedKey(self, K):
|
||||
return K.lower() in self._reserved
|
||||
|
||||
@@ -365,9 +372,15 @@ class Morsel(dict):
|
||||
}
|
||||
|
||||
def __setstate__(self, state):
|
||||
self._key = state['key']
|
||||
self._value = state['value']
|
||||
self._coded_value = state['coded_value']
|
||||
key = state['key']
|
||||
value = state['value']
|
||||
coded_value = state['coded_value']
|
||||
if _has_control_character(key, value, coded_value):
|
||||
raise CookieError("Control characters are not allowed in cookies "
|
||||
f"{key!r} {value!r} {coded_value!r}")
|
||||
self._key = key
|
||||
self._value = value
|
||||
self._coded_value = coded_value
|
||||
|
||||
def output(self, attrs=None, header="Set-Cookie:"):
|
||||
return "%s %s" % (header, self.OutputString(attrs))
|
||||
@@ -378,14 +391,21 @@ class Morsel(dict):
|
||||
return '<%s: %s>' % (self.__class__.__name__, self.OutputString())
|
||||
|
||||
def js_output(self, attrs=None):
|
||||
import base64
|
||||
# Print javascript
|
||||
output_string = self.OutputString(attrs)
|
||||
if _has_control_character(output_string):
|
||||
raise CookieError("Control characters are not allowed in cookies")
|
||||
# Base64-encode value to avoid template
|
||||
# injection in cookie values.
|
||||
output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii")
|
||||
return """
|
||||
<script type="text/javascript">
|
||||
<!-- begin hiding
|
||||
document.cookie = \"%s\";
|
||||
document.cookie = atob(\"%s\");
|
||||
// end hiding -->
|
||||
</script>
|
||||
""" % (self.OutputString(attrs).replace('"', r'\"'))
|
||||
""" % (output_encoded,)
|
||||
|
||||
def OutputString(self, attrs=None):
|
||||
# Build up our result
|
||||
|
||||
8
Lib/importlib/_bootstrap.py
vendored
8
Lib/importlib/_bootstrap.py
vendored
@@ -1375,6 +1375,14 @@ def _find_and_load(name, import_):
|
||||
# NOTE: because of this, initializing must be set *before*
|
||||
# putting the new module in sys.modules.
|
||||
_lock_unlock_module(name)
|
||||
else:
|
||||
# Verify the module is still in sys.modules. Another thread may have
|
||||
# removed it (due to import failure) between our sys.modules.get()
|
||||
# above and the _initializing check. If removed, we retry the import
|
||||
# to preserve normal semantics: the caller gets the exception from
|
||||
# the actual import failure rather than a synthetic error.
|
||||
if sys.modules.get(name) is not module:
|
||||
return _find_and_load(name, import_)
|
||||
|
||||
if module is None:
|
||||
message = f'import of {name} halted; None in sys.modules'
|
||||
|
||||
2
Lib/importlib/_bootstrap_external.py
vendored
2
Lib/importlib/_bootstrap_external.py
vendored
@@ -946,7 +946,7 @@ class FileLoader:
|
||||
|
||||
def get_data(self, path):
|
||||
"""Return the data from path as raw bytes."""
|
||||
if isinstance(self, (SourceLoader, ExtensionFileLoader)):
|
||||
if isinstance(self, (SourceLoader, SourcelessFileLoader, ExtensionFileLoader)):
|
||||
with _io.open_code(str(path)) as file:
|
||||
return file.read()
|
||||
else:
|
||||
|
||||
5
Lib/inspect.py
vendored
5
Lib/inspect.py
vendored
@@ -1,7 +1,7 @@
|
||||
"""Get useful information from live Python objects.
|
||||
|
||||
This module encapsulates the interface provided by the internal special
|
||||
attributes (co_*, im_*, tb_*, etc.) in a friendlier fashion.
|
||||
attributes (co_*, tb_*, etc.) in a friendlier fashion.
|
||||
It also provides some help for examining source code and class layout.
|
||||
|
||||
Here are some of the useful functions provided by this module:
|
||||
@@ -2660,11 +2660,12 @@ class Parameter:
|
||||
The annotation for the parameter if specified. If the
|
||||
parameter has no annotation, this attribute is set to
|
||||
`Parameter.empty`.
|
||||
* kind : str
|
||||
* kind
|
||||
Describes how argument values are bound to the parameter.
|
||||
Possible values: `Parameter.POSITIONAL_ONLY`,
|
||||
`Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
|
||||
`Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
|
||||
Every value has a `description` attribute describing meaning.
|
||||
"""
|
||||
|
||||
__slots__ = ('_name', '_kind', '_default', '_annotation')
|
||||
|
||||
39
Lib/logging/__init__.py
vendored
39
Lib/logging/__init__.py
vendored
@@ -1475,8 +1475,6 @@ class Logger(Filterer):
|
||||
level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels.
|
||||
There is no arbitrary limit to the depth of nesting.
|
||||
"""
|
||||
_tls = threading.local()
|
||||
|
||||
def __init__(self, name, level=NOTSET):
|
||||
"""
|
||||
Initialize the logger with a name and an optional level.
|
||||
@@ -1673,19 +1671,14 @@ class Logger(Filterer):
|
||||
This method is used for unpickled records received from a socket, as
|
||||
well as those created locally. Logger-level filtering is applied.
|
||||
"""
|
||||
if self._is_disabled():
|
||||
if self.disabled:
|
||||
return
|
||||
|
||||
self._tls.in_progress = True
|
||||
try:
|
||||
maybe_record = self.filter(record)
|
||||
if not maybe_record:
|
||||
return
|
||||
if isinstance(maybe_record, LogRecord):
|
||||
record = maybe_record
|
||||
self.callHandlers(record)
|
||||
finally:
|
||||
self._tls.in_progress = False
|
||||
maybe_record = self.filter(record)
|
||||
if not maybe_record:
|
||||
return
|
||||
if isinstance(maybe_record, LogRecord):
|
||||
record = maybe_record
|
||||
self.callHandlers(record)
|
||||
|
||||
def addHandler(self, hdlr):
|
||||
"""
|
||||
@@ -1773,7 +1766,7 @@ class Logger(Filterer):
|
||||
"""
|
||||
Is this logger enabled for level 'level'?
|
||||
"""
|
||||
if self._is_disabled():
|
||||
if self.disabled:
|
||||
return False
|
||||
|
||||
try:
|
||||
@@ -1823,11 +1816,6 @@ class Logger(Filterer):
|
||||
if isinstance(item, Logger) and item.parent is self and
|
||||
_hierlevel(item) == 1 + _hierlevel(item.parent))
|
||||
|
||||
def _is_disabled(self):
|
||||
# We need to use getattr as it will only be set the first time a log
|
||||
# message is recorded on any given thread
|
||||
return self.disabled or getattr(self._tls, 'in_progress', False)
|
||||
|
||||
def __repr__(self):
|
||||
level = getLevelName(self.getEffectiveLevel())
|
||||
return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)
|
||||
@@ -1864,9 +1852,9 @@ class LoggerAdapter(object):
|
||||
|
||||
def __init__(self, logger, extra=None, merge_extra=False):
|
||||
"""
|
||||
Initialize the adapter with a logger and a dict-like object which
|
||||
provides contextual information. This constructor signature allows
|
||||
easy stacking of LoggerAdapters, if so desired.
|
||||
Initialize the adapter with a logger and an optional dict-like object
|
||||
which provides contextual information. This constructor signature
|
||||
allows easy stacking of LoggerAdapters, if so desired.
|
||||
|
||||
You can effectively pass keyword arguments as shown in the
|
||||
following example:
|
||||
@@ -1897,8 +1885,9 @@ class LoggerAdapter(object):
|
||||
Normally, you'll only need to override this one method in a
|
||||
LoggerAdapter subclass for your specific needs.
|
||||
"""
|
||||
if self.merge_extra and "extra" in kwargs:
|
||||
kwargs["extra"] = {**self.extra, **kwargs["extra"]}
|
||||
if self.merge_extra and kwargs.get("extra") is not None:
|
||||
if self.extra is not None:
|
||||
kwargs["extra"] = {**self.extra, **kwargs["extra"]}
|
||||
else:
|
||||
kwargs["extra"] = self.extra
|
||||
return msg, kwargs
|
||||
|
||||
14
Lib/logging/config.py
vendored
14
Lib/logging/config.py
vendored
@@ -865,6 +865,8 @@ class DictConfigurator(BaseConfigurator):
|
||||
else:
|
||||
factory = klass
|
||||
kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))}
|
||||
# When deprecation ends for using the 'strm' parameter, remove the
|
||||
# "except TypeError ..."
|
||||
try:
|
||||
result = factory(**kwargs)
|
||||
except TypeError as te:
|
||||
@@ -876,6 +878,15 @@ class DictConfigurator(BaseConfigurator):
|
||||
#(e.g. by Django)
|
||||
kwargs['strm'] = kwargs.pop('stream')
|
||||
result = factory(**kwargs)
|
||||
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"Support for custom logging handlers with the 'strm' argument "
|
||||
"is deprecated and scheduled for removal in Python 3.16. "
|
||||
"Define handlers with the 'stream' argument instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if formatter:
|
||||
result.setFormatter(formatter)
|
||||
if level is not None:
|
||||
@@ -1006,7 +1017,8 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
|
||||
A simple TCP socket-based logging config receiver.
|
||||
"""
|
||||
|
||||
allow_reuse_address = 1
|
||||
allow_reuse_address = True
|
||||
allow_reuse_port = False
|
||||
|
||||
def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
|
||||
handler=None, ready=None, verify=None):
|
||||
|
||||
24
Lib/logging/handlers.py
vendored
24
Lib/logging/handlers.py
vendored
@@ -196,7 +196,11 @@ class RotatingFileHandler(BaseRotatingHandler):
|
||||
if self.stream is None: # delay was set...
|
||||
self.stream = self._open()
|
||||
if self.maxBytes > 0: # are we rolling over?
|
||||
pos = self.stream.tell()
|
||||
try:
|
||||
pos = self.stream.tell()
|
||||
except io.UnsupportedOperation:
|
||||
# gh-143237: Never rollover a named pipe.
|
||||
return False
|
||||
if not pos:
|
||||
# gh-116263: Never rollover an empty file
|
||||
return False
|
||||
@@ -855,7 +859,7 @@ class SysLogHandler(logging.Handler):
|
||||
}
|
||||
|
||||
def __init__(self, address=('localhost', SYSLOG_UDP_PORT),
|
||||
facility=LOG_USER, socktype=None):
|
||||
facility=LOG_USER, socktype=None, timeout=None):
|
||||
"""
|
||||
Initialize a handler.
|
||||
|
||||
@@ -872,6 +876,7 @@ class SysLogHandler(logging.Handler):
|
||||
self.address = address
|
||||
self.facility = facility
|
||||
self.socktype = socktype
|
||||
self.timeout = timeout
|
||||
self.socket = None
|
||||
self.createSocket()
|
||||
|
||||
@@ -933,6 +938,8 @@ class SysLogHandler(logging.Handler):
|
||||
err = sock = None
|
||||
try:
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
if self.timeout:
|
||||
sock.settimeout(self.timeout)
|
||||
if socktype == socket.SOCK_STREAM:
|
||||
sock.connect(sa)
|
||||
break
|
||||
@@ -1529,6 +1536,19 @@ class QueueListener(object):
|
||||
self._thread = None
|
||||
self.respect_handler_level = respect_handler_level
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
For use as a context manager. Starts the listener.
|
||||
"""
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
"""
|
||||
For use as a context manager. Stops the listener.
|
||||
"""
|
||||
self.stop()
|
||||
|
||||
def dequeue(self, block):
|
||||
"""
|
||||
Dequeue a record and return it, optionally blocking.
|
||||
|
||||
118
Lib/multiprocessing/connection.py
vendored
118
Lib/multiprocessing/connection.py
vendored
@@ -11,13 +11,12 @@ __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
|
||||
|
||||
import errno
|
||||
import io
|
||||
import itertools
|
||||
import os
|
||||
import sys
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import tempfile
|
||||
import itertools
|
||||
|
||||
|
||||
from . import util
|
||||
@@ -39,11 +38,14 @@ except ImportError:
|
||||
#
|
||||
#
|
||||
|
||||
BUFSIZE = 8192
|
||||
# 64 KiB is the default PIPE buffer size of most POSIX platforms.
|
||||
BUFSIZE = 64 * 1024
|
||||
|
||||
# A very generous timeout when it comes to local connections...
|
||||
CONNECTION_TIMEOUT = 20.
|
||||
|
||||
_mmap_counter = itertools.count()
|
||||
_MAX_PIPE_ATTEMPTS = 100
|
||||
|
||||
default_family = 'AF_INET'
|
||||
families = ['AF_INET']
|
||||
@@ -74,10 +76,14 @@ def arbitrary_address(family):
|
||||
if family == 'AF_INET':
|
||||
return ('localhost', 0)
|
||||
elif family == 'AF_UNIX':
|
||||
return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir())
|
||||
# NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
|
||||
# mktemp-style ToC vs ToU concern is not important; bind() surfaces
|
||||
# the extremely unlikely collision as EADDRINUSE.
|
||||
return os.path.join(util.get_temp_dir(),
|
||||
f'sock-{os.urandom(6).hex()}')
|
||||
elif family == 'AF_PIPE':
|
||||
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
|
||||
(os.getpid(), next(_mmap_counter)), dir="")
|
||||
return (r'\\.\pipe\pyc-%d-%d-%s' %
|
||||
(os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
|
||||
else:
|
||||
raise ValueError('unrecognized family')
|
||||
|
||||
@@ -179,6 +185,10 @@ class _ConnectionBase:
|
||||
finally:
|
||||
self._handle = None
|
||||
|
||||
def _detach(self):
|
||||
"""Stop managing the underlying file descriptor or handle."""
|
||||
self._handle = None
|
||||
|
||||
def send_bytes(self, buf, offset=0, size=None):
|
||||
"""Send the bytes data from a bytes-like object"""
|
||||
self._check_closed()
|
||||
@@ -316,22 +326,32 @@ if _winapi:
|
||||
try:
|
||||
ov, err = _winapi.ReadFile(self._handle, bsize,
|
||||
overlapped=True)
|
||||
|
||||
sentinel = object()
|
||||
return_value = sentinel
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
waitres = _winapi.WaitForMultipleObjects(
|
||||
[ov.event], False, INFINITE)
|
||||
assert waitres == WAIT_OBJECT_0
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
waitres = _winapi.WaitForMultipleObjects(
|
||||
[ov.event], False, INFINITE)
|
||||
assert waitres == WAIT_OBJECT_0
|
||||
except:
|
||||
ov.cancel()
|
||||
raise
|
||||
finally:
|
||||
nread, err = ov.GetOverlappedResult(True)
|
||||
if err == 0:
|
||||
f = io.BytesIO()
|
||||
f.write(ov.getbuffer())
|
||||
return_value = f
|
||||
elif err == _winapi.ERROR_MORE_DATA:
|
||||
return_value = self._get_more_data(ov, maxsize)
|
||||
except:
|
||||
ov.cancel()
|
||||
raise
|
||||
finally:
|
||||
nread, err = ov.GetOverlappedResult(True)
|
||||
if err == 0:
|
||||
f = io.BytesIO()
|
||||
f.write(ov.getbuffer())
|
||||
return f
|
||||
elif err == _winapi.ERROR_MORE_DATA:
|
||||
return self._get_more_data(ov, maxsize)
|
||||
if return_value is sentinel:
|
||||
raise
|
||||
|
||||
if return_value is not sentinel:
|
||||
return return_value
|
||||
except OSError as e:
|
||||
if e.winerror == _winapi.ERROR_BROKEN_PIPE:
|
||||
raise EOFError
|
||||
@@ -392,7 +412,8 @@ class Connection(_ConnectionBase):
|
||||
handle = self._handle
|
||||
remaining = size
|
||||
while remaining > 0:
|
||||
chunk = read(handle, remaining)
|
||||
to_read = min(BUFSIZE, remaining)
|
||||
chunk = read(handle, to_read)
|
||||
n = len(chunk)
|
||||
if n == 0:
|
||||
if remaining == size:
|
||||
@@ -455,17 +476,29 @@ class Listener(object):
|
||||
def __init__(self, address=None, family=None, backlog=1, authkey=None):
|
||||
family = family or (address and address_type(address)) \
|
||||
or default_family
|
||||
address = address or arbitrary_address(family)
|
||||
|
||||
_validate_family(family)
|
||||
if family == 'AF_PIPE':
|
||||
self._listener = PipeListener(address, backlog)
|
||||
else:
|
||||
self._listener = SocketListener(address, family, backlog)
|
||||
|
||||
if authkey is not None and not isinstance(authkey, bytes):
|
||||
raise TypeError('authkey should be a byte string')
|
||||
|
||||
if family == 'AF_PIPE':
|
||||
if address:
|
||||
self._listener = PipeListener(address, backlog)
|
||||
else:
|
||||
for attempts in itertools.count():
|
||||
address = arbitrary_address(family)
|
||||
try:
|
||||
self._listener = PipeListener(address, backlog)
|
||||
break
|
||||
except OSError as e:
|
||||
if attempts >= _MAX_PIPE_ATTEMPTS:
|
||||
raise
|
||||
if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
|
||||
_winapi.ERROR_ACCESS_DENIED):
|
||||
raise
|
||||
else:
|
||||
address = address or arbitrary_address(family)
|
||||
self._listener = SocketListener(address, family, backlog)
|
||||
|
||||
self._authkey = authkey
|
||||
|
||||
def accept(self):
|
||||
@@ -553,7 +586,6 @@ else:
|
||||
'''
|
||||
Returns pair of connection objects at either end of a pipe
|
||||
'''
|
||||
address = arbitrary_address('AF_PIPE')
|
||||
if duplex:
|
||||
openmode = _winapi.PIPE_ACCESS_DUPLEX
|
||||
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
|
||||
@@ -563,15 +595,25 @@ else:
|
||||
access = _winapi.GENERIC_WRITE
|
||||
obsize, ibsize = 0, BUFSIZE
|
||||
|
||||
h1 = _winapi.CreateNamedPipe(
|
||||
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
|
||||
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
|
||||
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
|
||||
_winapi.PIPE_WAIT,
|
||||
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
|
||||
# default security descriptor: the handle cannot be inherited
|
||||
_winapi.NULL
|
||||
)
|
||||
for attempts in itertools.count():
|
||||
address = arbitrary_address('AF_PIPE')
|
||||
try:
|
||||
h1 = _winapi.CreateNamedPipe(
|
||||
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
|
||||
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
|
||||
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
|
||||
_winapi.PIPE_WAIT,
|
||||
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
|
||||
# default security descriptor: the handle cannot be inherited
|
||||
_winapi.NULL
|
||||
)
|
||||
break
|
||||
except OSError as e:
|
||||
if attempts >= _MAX_PIPE_ATTEMPTS:
|
||||
raise
|
||||
if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
|
||||
_winapi.ERROR_ACCESS_DENIED):
|
||||
raise
|
||||
h2 = _winapi.CreateFile(
|
||||
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
|
||||
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
|
||||
|
||||
36
Lib/multiprocessing/context.py
vendored
36
Lib/multiprocessing/context.py
vendored
@@ -145,7 +145,13 @@ class BaseContext(object):
|
||||
'''Check whether this is a fake forked process in a frozen executable.
|
||||
If so then run code specified by commandline and exit.
|
||||
'''
|
||||
if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False):
|
||||
# gh-140814: allow_none=True avoids locking in the default start
|
||||
# method, which would cause a later set_start_method() to fail.
|
||||
# None is safe to pass through: spawn.freeze_support()
|
||||
# independently detects whether this process is a spawned
|
||||
# child, so the start method check here is only an optimization.
|
||||
if (getattr(sys, 'frozen', False)
|
||||
and self.get_start_method(allow_none=True) in ('spawn', None)):
|
||||
from .spawn import freeze_support
|
||||
freeze_support()
|
||||
|
||||
@@ -167,7 +173,7 @@ class BaseContext(object):
|
||||
'''
|
||||
# This is undocumented. In previous versions of multiprocessing
|
||||
# its only effect was to make socket objects inheritable on Windows.
|
||||
from . import connection
|
||||
from . import connection # noqa: F401
|
||||
|
||||
def set_executable(self, executable):
|
||||
'''Sets the path to a python.exe or pythonw.exe binary used to run
|
||||
@@ -259,13 +265,12 @@ class DefaultContext(BaseContext):
|
||||
|
||||
def get_all_start_methods(self):
|
||||
"""Returns a list of the supported start methods, default first."""
|
||||
if sys.platform == 'win32':
|
||||
return ['spawn']
|
||||
else:
|
||||
methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn']
|
||||
if reduction.HAVE_SEND_HANDLE:
|
||||
methods.append('forkserver')
|
||||
return methods
|
||||
default = self._default_context.get_start_method()
|
||||
start_method_names = [default]
|
||||
start_method_names.extend(
|
||||
name for name in _concrete_contexts if name != default
|
||||
)
|
||||
return start_method_names
|
||||
|
||||
|
||||
#
|
||||
@@ -320,14 +325,15 @@ if sys.platform != 'win32':
|
||||
'spawn': SpawnContext(),
|
||||
'forkserver': ForkServerContext(),
|
||||
}
|
||||
if sys.platform == 'darwin':
|
||||
# bpo-33725: running arbitrary code after fork() is no longer reliable
|
||||
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
|
||||
_default_context = DefaultContext(_concrete_contexts['spawn'])
|
||||
# bpo-33725: running arbitrary code after fork() is no longer reliable
|
||||
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
|
||||
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
|
||||
if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
|
||||
_default_context = DefaultContext(_concrete_contexts['forkserver'])
|
||||
else:
|
||||
_default_context = DefaultContext(_concrete_contexts['fork'])
|
||||
_default_context = DefaultContext(_concrete_contexts['spawn'])
|
||||
|
||||
else:
|
||||
else: # Windows
|
||||
|
||||
class SpawnProcess(process.BaseProcess):
|
||||
_start_method = 'spawn'
|
||||
|
||||
2
Lib/multiprocessing/dummy/__init__.py
vendored
2
Lib/multiprocessing/dummy/__init__.py
vendored
@@ -33,7 +33,7 @@ from queue import Queue
|
||||
|
||||
class DummyProcess(threading.Thread):
|
||||
|
||||
def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
|
||||
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None):
|
||||
threading.Thread.__init__(self, group, target, name, args, kwargs)
|
||||
self._pid = None
|
||||
self._children = weakref.WeakKeyDictionary()
|
||||
|
||||
96
Lib/multiprocessing/forkserver.py
vendored
96
Lib/multiprocessing/forkserver.py
vendored
@@ -9,6 +9,7 @@ import sys
|
||||
import threading
|
||||
import warnings
|
||||
|
||||
from . import AuthenticationError
|
||||
from . import connection
|
||||
from . import process
|
||||
from .context import reduction
|
||||
@@ -25,6 +26,7 @@ __all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
|
||||
|
||||
MAXFDS_TO_SEND = 256
|
||||
SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
|
||||
_AUTHKEY_LEN = 32 # <= PIPEBUF so it fits a single write to an empty pipe.
|
||||
|
||||
#
|
||||
# Forkserver class
|
||||
@@ -33,6 +35,7 @@ SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
|
||||
class ForkServer(object):
|
||||
|
||||
def __init__(self):
|
||||
self._forkserver_authkey = None
|
||||
self._forkserver_address = None
|
||||
self._forkserver_alive_fd = None
|
||||
self._forkserver_pid = None
|
||||
@@ -59,6 +62,7 @@ class ForkServer(object):
|
||||
if not util.is_abstract_socket_namespace(self._forkserver_address):
|
||||
os.unlink(self._forkserver_address)
|
||||
self._forkserver_address = None
|
||||
self._forkserver_authkey = None
|
||||
|
||||
def set_forkserver_preload(self, modules_names):
|
||||
'''Set list of module names to try to load in forkserver process.'''
|
||||
@@ -83,6 +87,7 @@ class ForkServer(object):
|
||||
process data.
|
||||
'''
|
||||
self.ensure_running()
|
||||
assert self._forkserver_authkey
|
||||
if len(fds) + 4 >= MAXFDS_TO_SEND:
|
||||
raise ValueError('too many fds')
|
||||
with socket.socket(socket.AF_UNIX) as client:
|
||||
@@ -93,6 +98,18 @@ class ForkServer(object):
|
||||
resource_tracker.getfd()]
|
||||
allfds += fds
|
||||
try:
|
||||
client.setblocking(True)
|
||||
wrapped_client = connection.Connection(client.fileno())
|
||||
# The other side of this exchange happens in the child as
|
||||
# implemented in main().
|
||||
try:
|
||||
connection.answer_challenge(
|
||||
wrapped_client, self._forkserver_authkey)
|
||||
connection.deliver_challenge(
|
||||
wrapped_client, self._forkserver_authkey)
|
||||
finally:
|
||||
wrapped_client._detach()
|
||||
del wrapped_client
|
||||
reduction.sendfds(client, allfds)
|
||||
return parent_r, parent_w
|
||||
except:
|
||||
@@ -120,20 +137,30 @@ class ForkServer(object):
|
||||
return
|
||||
# dead, launch it again
|
||||
os.close(self._forkserver_alive_fd)
|
||||
self._forkserver_authkey = None
|
||||
self._forkserver_address = None
|
||||
self._forkserver_alive_fd = None
|
||||
self._forkserver_pid = None
|
||||
|
||||
cmd = ('from multiprocessing.forkserver import main; ' +
|
||||
'main(%d, %d, %r, **%r)')
|
||||
# gh-144503: sys_argv is passed as real argv elements after the
|
||||
# ``-c cmd`` rather than repr'd into main_kws so that a large
|
||||
# parent sys.argv cannot push the single ``-c`` command string
|
||||
# over the OS per-argument length limit (MAX_ARG_STRLEN on Linux).
|
||||
# The child sees them as sys.argv[1:].
|
||||
cmd = ('import sys; '
|
||||
'from multiprocessing.forkserver import main; '
|
||||
'main(%d, %d, %r, sys_argv=sys.argv[1:], **%r)')
|
||||
|
||||
main_kws = {}
|
||||
sys_argv = None
|
||||
if self._preload_modules:
|
||||
data = spawn.get_preparation_data('ignore')
|
||||
if 'sys_path' in data:
|
||||
main_kws['sys_path'] = data['sys_path']
|
||||
if 'init_main_from_path' in data:
|
||||
main_kws['main_path'] = data['init_main_from_path']
|
||||
if 'sys_argv' in data:
|
||||
sys_argv = data['sys_argv']
|
||||
|
||||
with socket.socket(socket.AF_UNIX) as listener:
|
||||
address = connection.arbitrary_address('AF_UNIX')
|
||||
@@ -145,19 +172,33 @@ class ForkServer(object):
|
||||
# all client processes own the write end of the "alive" pipe;
|
||||
# when they all terminate the read end becomes ready.
|
||||
alive_r, alive_w = os.pipe()
|
||||
# A short lived pipe to initialize the forkserver authkey.
|
||||
authkey_r, authkey_w = os.pipe()
|
||||
try:
|
||||
fds_to_pass = [listener.fileno(), alive_r]
|
||||
fds_to_pass = [listener.fileno(), alive_r, authkey_r]
|
||||
main_kws['authkey_r'] = authkey_r
|
||||
cmd %= (listener.fileno(), alive_r, self._preload_modules,
|
||||
main_kws)
|
||||
exe = spawn.get_executable()
|
||||
args = [exe] + util._args_from_interpreter_flags()
|
||||
args += ['-c', cmd]
|
||||
if sys_argv is not None:
|
||||
args += sys_argv
|
||||
pid = util.spawnv_passfds(exe, args, fds_to_pass)
|
||||
except:
|
||||
os.close(alive_w)
|
||||
os.close(authkey_w)
|
||||
raise
|
||||
finally:
|
||||
os.close(alive_r)
|
||||
os.close(authkey_r)
|
||||
# Authenticate our control socket to prevent access from
|
||||
# processes we have not shared this key with.
|
||||
try:
|
||||
self._forkserver_authkey = os.urandom(_AUTHKEY_LEN)
|
||||
os.write(authkey_w, self._forkserver_authkey)
|
||||
finally:
|
||||
os.close(authkey_w)
|
||||
self._forkserver_address = address
|
||||
self._forkserver_alive_fd = alive_w
|
||||
self._forkserver_pid = pid
|
||||
@@ -166,9 +207,21 @@ class ForkServer(object):
|
||||
#
|
||||
#
|
||||
|
||||
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
|
||||
'''Run forkserver.'''
|
||||
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None,
|
||||
*, sys_argv=None, authkey_r=None):
|
||||
"""Run forkserver."""
|
||||
if authkey_r is not None:
|
||||
try:
|
||||
authkey = os.read(authkey_r, _AUTHKEY_LEN)
|
||||
assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}'
|
||||
finally:
|
||||
os.close(authkey_r)
|
||||
else:
|
||||
authkey = b''
|
||||
|
||||
if preload:
|
||||
if sys_argv is not None:
|
||||
sys.argv[:] = sys_argv
|
||||
if sys_path is not None:
|
||||
sys.path[:] = sys_path
|
||||
if '__main__' in preload and main_path is not None:
|
||||
@@ -262,8 +315,24 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
|
||||
if listener in rfds:
|
||||
# Incoming fork request
|
||||
with listener.accept()[0] as s:
|
||||
# Receive fds from client
|
||||
fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
|
||||
try:
|
||||
if authkey:
|
||||
wrapped_s = connection.Connection(s.fileno())
|
||||
# The other side of this exchange happens in
|
||||
# in connect_to_new_process().
|
||||
try:
|
||||
connection.deliver_challenge(
|
||||
wrapped_s, authkey)
|
||||
connection.answer_challenge(
|
||||
wrapped_s, authkey)
|
||||
finally:
|
||||
wrapped_s._detach()
|
||||
del wrapped_s
|
||||
# Receive fds from client
|
||||
fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
|
||||
except (EOFError, BrokenPipeError, AuthenticationError):
|
||||
s.close()
|
||||
continue
|
||||
if len(fds) > MAXFDS_TO_SEND:
|
||||
raise RuntimeError(
|
||||
"Too many ({0:n}) fds to send".format(
|
||||
@@ -331,13 +400,14 @@ def _serve_one(child_r, fds, unused_fds, handlers):
|
||||
#
|
||||
|
||||
def read_signed(fd):
|
||||
data = b''
|
||||
length = SIGNED_STRUCT.size
|
||||
while len(data) < length:
|
||||
s = os.read(fd, length - len(data))
|
||||
if not s:
|
||||
data = bytearray(SIGNED_STRUCT.size)
|
||||
unread = memoryview(data)
|
||||
while unread:
|
||||
count = os.readinto(fd, unread)
|
||||
if count == 0:
|
||||
raise EOFError('unexpected EOF')
|
||||
data += s
|
||||
unread = unread[count:]
|
||||
|
||||
return SIGNED_STRUCT.unpack(data)[0]
|
||||
|
||||
def write_signed(fd, n):
|
||||
|
||||
59
Lib/multiprocessing/managers.py
vendored
59
Lib/multiprocessing/managers.py
vendored
@@ -18,6 +18,7 @@ import sys
|
||||
import threading
|
||||
import signal
|
||||
import array
|
||||
import collections.abc
|
||||
import queue
|
||||
import time
|
||||
import types
|
||||
@@ -1058,12 +1059,14 @@ class IteratorProxy(BaseProxy):
|
||||
|
||||
|
||||
class AcquirerProxy(BaseProxy):
|
||||
_exposed_ = ('acquire', 'release')
|
||||
_exposed_ = ('acquire', 'release', 'locked')
|
||||
def acquire(self, blocking=True, timeout=None):
|
||||
args = (blocking,) if timeout is None else (blocking, timeout)
|
||||
return self._callmethod('acquire', args)
|
||||
def release(self):
|
||||
return self._callmethod('release')
|
||||
def locked(self):
|
||||
return self._callmethod('locked')
|
||||
def __enter__(self):
|
||||
return self._callmethod('acquire')
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
@@ -1071,7 +1074,7 @@ class AcquirerProxy(BaseProxy):
|
||||
|
||||
|
||||
class ConditionProxy(AcquirerProxy):
|
||||
_exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all')
|
||||
_exposed_ = ('acquire', 'release', 'locked', 'wait', 'notify', 'notify_all')
|
||||
def wait(self, timeout=None):
|
||||
return self._callmethod('wait', (timeout,))
|
||||
def notify(self, n=1):
|
||||
@@ -1159,10 +1162,10 @@ class ValueProxy(BaseProxy):
|
||||
|
||||
|
||||
BaseListProxy = MakeProxyType('BaseListProxy', (
|
||||
'__add__', '__contains__', '__delitem__', '__getitem__', '__len__',
|
||||
'__mul__', '__reversed__', '__rmul__', '__setitem__',
|
||||
'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
|
||||
'reverse', 'sort', '__imul__'
|
||||
'__add__', '__contains__', '__delitem__', '__getitem__', '__imul__',
|
||||
'__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__',
|
||||
'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop',
|
||||
'remove', 'reverse', 'sort',
|
||||
))
|
||||
class ListProxy(BaseListProxy):
|
||||
def __iadd__(self, value):
|
||||
@@ -1174,18 +1177,55 @@ class ListProxy(BaseListProxy):
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
collections.abc.MutableSequence.register(BaseListProxy)
|
||||
|
||||
_BaseDictProxy = MakeProxyType('DictProxy', (
|
||||
'__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
|
||||
'__setitem__', 'clear', 'copy', 'get', 'items',
|
||||
_BaseDictProxy = MakeProxyType('_BaseDictProxy', (
|
||||
'__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__',
|
||||
'__len__', '__or__', '__reversed__', '__ror__',
|
||||
'__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items',
|
||||
'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
|
||||
))
|
||||
_BaseDictProxy._method_to_typeid_ = {
|
||||
'__iter__': 'Iterator',
|
||||
}
|
||||
class DictProxy(_BaseDictProxy):
|
||||
def __ior__(self, value):
|
||||
self._callmethod('__ior__', (value,))
|
||||
return self
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
collections.abc.MutableMapping.register(_BaseDictProxy)
|
||||
|
||||
_BaseSetProxy = MakeProxyType("_BaseSetProxy", (
|
||||
'__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
|
||||
'__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
|
||||
'__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
|
||||
'__ge__', '__gt__', '__le__', '__lt__',
|
||||
'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
|
||||
'intersection', 'intersection_update', 'isdisjoint', 'issubset',
|
||||
'issuperset', 'pop', 'remove', 'symmetric_difference',
|
||||
'symmetric_difference_update', 'union', 'update',
|
||||
))
|
||||
|
||||
class SetProxy(_BaseSetProxy):
|
||||
def __ior__(self, value):
|
||||
self._callmethod('__ior__', (value,))
|
||||
return self
|
||||
def __iand__(self, value):
|
||||
self._callmethod('__iand__', (value,))
|
||||
return self
|
||||
def __ixor__(self, value):
|
||||
self._callmethod('__ixor__', (value,))
|
||||
return self
|
||||
def __isub__(self, value):
|
||||
self._callmethod('__isub__', (value,))
|
||||
return self
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
collections.abc.MutableMapping.register(_BaseSetProxy)
|
||||
|
||||
|
||||
ArrayProxy = MakeProxyType('ArrayProxy', (
|
||||
'__len__', '__getitem__', '__setitem__'
|
||||
@@ -1237,6 +1277,7 @@ SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
|
||||
SyncManager.register('Pool', pool.Pool, PoolProxy)
|
||||
SyncManager.register('list', list, ListProxy)
|
||||
SyncManager.register('dict', dict, DictProxy)
|
||||
SyncManager.register('set', set, SetProxy)
|
||||
SyncManager.register('Value', Value, ValueProxy)
|
||||
SyncManager.register('Array', Array, ArrayProxy)
|
||||
SyncManager.register('Namespace', Namespace, NamespaceProxy)
|
||||
|
||||
15
Lib/multiprocessing/popen_fork.py
vendored
15
Lib/multiprocessing/popen_fork.py
vendored
@@ -54,6 +54,9 @@ class Popen(object):
|
||||
if self.wait(timeout=0.1) is None:
|
||||
raise
|
||||
|
||||
def interrupt(self):
|
||||
self._send_signal(signal.SIGINT)
|
||||
|
||||
def terminate(self):
|
||||
self._send_signal(signal.SIGTERM)
|
||||
|
||||
@@ -64,7 +67,17 @@ class Popen(object):
|
||||
code = 1
|
||||
parent_r, child_w = os.pipe()
|
||||
child_r, parent_w = os.pipe()
|
||||
self.pid = os.fork()
|
||||
# gh-146313: Tell the resource tracker's at-fork handler to keep
|
||||
# the inherited pipe fd so this child reuses the parent's tracker
|
||||
# (gh-80849) rather than closing it and launching its own.
|
||||
from .resource_tracker import _fork_intent
|
||||
_fork_intent.preserve_fd = True
|
||||
try:
|
||||
self.pid = os.fork()
|
||||
finally:
|
||||
# Reset in both parent and child so the flag does not leak
|
||||
# into a subsequent raw os.fork() or nested Process launch.
|
||||
_fork_intent.preserve_fd = False
|
||||
if self.pid == 0:
|
||||
try:
|
||||
atexit._clear()
|
||||
|
||||
11
Lib/multiprocessing/process.py
vendored
11
Lib/multiprocessing/process.py
vendored
@@ -77,7 +77,7 @@ class BaseProcess(object):
|
||||
def _Popen(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
|
||||
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None,
|
||||
*, daemon=None):
|
||||
assert group is None, 'group argument must be None for now'
|
||||
count = next(_process_counter)
|
||||
@@ -89,7 +89,7 @@ class BaseProcess(object):
|
||||
self._closed = False
|
||||
self._target = target
|
||||
self._args = tuple(args)
|
||||
self._kwargs = dict(kwargs)
|
||||
self._kwargs = dict(kwargs) if kwargs else {}
|
||||
self._name = name or type(self).__name__ + '-' + \
|
||||
':'.join(str(i) for i in self._identity)
|
||||
if daemon is not None:
|
||||
@@ -125,6 +125,13 @@ class BaseProcess(object):
|
||||
del self._target, self._args, self._kwargs
|
||||
_children.add(self)
|
||||
|
||||
def interrupt(self):
|
||||
'''
|
||||
Terminate process; sends SIGINT signal
|
||||
'''
|
||||
self._check_closed()
|
||||
self._popen.interrupt()
|
||||
|
||||
def terminate(self):
|
||||
'''
|
||||
Terminate process; sends SIGTERM signal or uses TerminateProcess()
|
||||
|
||||
2
Lib/multiprocessing/queues.py
vendored
2
Lib/multiprocessing/queues.py
vendored
@@ -121,7 +121,7 @@ class Queue(object):
|
||||
|
||||
def qsize(self):
|
||||
# Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
|
||||
return self._maxsize - self._sem._semlock._get_value()
|
||||
return self._maxsize - self._sem.get_value()
|
||||
|
||||
def empty(self):
|
||||
return not self._poll()
|
||||
|
||||
12
Lib/multiprocessing/reduction.py
vendored
12
Lib/multiprocessing/reduction.py
vendored
@@ -139,15 +139,12 @@ else:
|
||||
__all__ += ['DupFd', 'sendfds', 'recvfds']
|
||||
import array
|
||||
|
||||
# On MacOSX we should acknowledge receipt of fds -- see Issue14669
|
||||
ACKNOWLEDGE = sys.platform == 'darwin'
|
||||
|
||||
def sendfds(sock, fds):
|
||||
'''Send an array of fds over an AF_UNIX socket.'''
|
||||
fds = array.array('i', fds)
|
||||
msg = bytes([len(fds) % 256])
|
||||
sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
|
||||
if ACKNOWLEDGE and sock.recv(1) != b'A':
|
||||
if sock.recv(1) != b'A':
|
||||
raise RuntimeError('did not receive acknowledgement of fd')
|
||||
|
||||
def recvfds(sock, size):
|
||||
@@ -158,8 +155,11 @@ else:
|
||||
if not msg and not ancdata:
|
||||
raise EOFError
|
||||
try:
|
||||
if ACKNOWLEDGE:
|
||||
sock.send(b'A')
|
||||
# We send/recv an Ack byte after the fds to work around an old
|
||||
# macOS bug; it isn't clear if this is still required but it
|
||||
# makes unit testing fd sending easier.
|
||||
# See: https://github.com/python/cpython/issues/58874
|
||||
sock.send(b'A') # Acknowledge
|
||||
if len(ancdata) != 1:
|
||||
raise RuntimeError('received %d items of ancdata' %
|
||||
len(ancdata))
|
||||
|
||||
97
Lib/multiprocessing/resource_tracker.py
vendored
97
Lib/multiprocessing/resource_tracker.py
vendored
@@ -20,6 +20,7 @@ import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import warnings
|
||||
from collections import deque
|
||||
|
||||
@@ -51,12 +52,8 @@ if os.name == 'posix':
|
||||
# absence of POSIX named semaphores. In that case, no named semaphores were
|
||||
# ever opened, so no cleanup would be necessary.
|
||||
if hasattr(_multiprocessing, 'sem_unlink'):
|
||||
_CLEANUP_FUNCS.update({
|
||||
'semaphore': _multiprocessing.sem_unlink,
|
||||
})
|
||||
_CLEANUP_FUNCS.update({
|
||||
'shared_memory': _posixshmem.shm_unlink,
|
||||
})
|
||||
_CLEANUP_FUNCS['semaphore'] = _multiprocessing.sem_unlink
|
||||
_CLEANUP_FUNCS['shared_memory'] = _posixshmem.shm_unlink
|
||||
|
||||
|
||||
class ReentrantCallError(RuntimeError):
|
||||
@@ -79,6 +76,10 @@ class ResourceTracker(object):
|
||||
# The reader should understand all formats.
|
||||
self._use_simple_format = True
|
||||
|
||||
# Set to True by _stop_locked() if the waitpid polling loop ran to
|
||||
# its timeout without reaping the tracker. Exposed for tests.
|
||||
self._waitpid_timed_out = False
|
||||
|
||||
def _reentrant_call_error(self):
|
||||
# gh-109629: this happens if an explicit call to the ResourceTracker
|
||||
# gets interrupted by a garbage collection, invoking a finalizer (*)
|
||||
@@ -91,16 +92,51 @@ class ResourceTracker(object):
|
||||
# making sure child processess are cleaned before ResourceTracker
|
||||
# gets destructed.
|
||||
# see https://github.com/python/cpython/issues/88887
|
||||
self._stop(use_blocking_lock=False)
|
||||
# gh-146313: use a timeout to avoid deadlocking if a forked child
|
||||
# still holds the pipe's write end open.
|
||||
self._stop(use_blocking_lock=False, wait_timeout=1.0)
|
||||
|
||||
def _stop(self, use_blocking_lock=True):
|
||||
def _after_fork_in_child(self):
|
||||
# gh-146313: Called in the child right after os.fork().
|
||||
#
|
||||
# The tracker process is a child of the *parent*, not of us, so we
|
||||
# could never waitpid() it anyway. Clearing _pid means our __del__
|
||||
# becomes a no-op (the early return for _pid is None).
|
||||
#
|
||||
# Whether we keep the inherited _fd depends on who forked us:
|
||||
#
|
||||
# - multiprocessing.Process with the 'fork' start method sets
|
||||
# _fork_intent.preserve_fd before forking. The child keeps the
|
||||
# fd and reuses the parent's tracker (gh-80849). This is safe
|
||||
# because multiprocessing's atexit handler joins all children
|
||||
# before the parent's __del__ runs, so by then the fd copies
|
||||
# are gone and the parent can reap the tracker promptly.
|
||||
#
|
||||
# - A raw os.fork() leaves the flag unset. We close the fd in the child after forking so
|
||||
# the parent's __del__ can reap the tracker without waiting
|
||||
# for the child to exit. If we later need a tracker, ensure_running()
|
||||
# will launch a fresh one.
|
||||
self._lock._at_fork_reinit()
|
||||
self._reentrant_messages.clear()
|
||||
self._pid = None
|
||||
self._exitcode = None
|
||||
if (self._fd is not None and
|
||||
not getattr(_fork_intent, 'preserve_fd', False)):
|
||||
fd = self._fd
|
||||
self._fd = None
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _stop(self, use_blocking_lock=True, wait_timeout=None):
|
||||
if use_blocking_lock:
|
||||
with self._lock:
|
||||
self._stop_locked()
|
||||
self._stop_locked(wait_timeout=wait_timeout)
|
||||
else:
|
||||
acquired = self._lock.acquire(blocking=False)
|
||||
try:
|
||||
self._stop_locked()
|
||||
self._stop_locked(wait_timeout=wait_timeout)
|
||||
finally:
|
||||
if acquired:
|
||||
self._lock.release()
|
||||
@@ -110,6 +146,10 @@ class ResourceTracker(object):
|
||||
close=os.close,
|
||||
waitpid=os.waitpid,
|
||||
waitstatus_to_exitcode=os.waitstatus_to_exitcode,
|
||||
monotonic=time.monotonic,
|
||||
sleep=time.sleep,
|
||||
WNOHANG=getattr(os, 'WNOHANG', None),
|
||||
wait_timeout=None,
|
||||
):
|
||||
# This shouldn't happen (it might when called by a finalizer)
|
||||
# so we check for it anyway.
|
||||
@@ -126,7 +166,30 @@ class ResourceTracker(object):
|
||||
self._fd = None
|
||||
|
||||
try:
|
||||
_, status = waitpid(self._pid, 0)
|
||||
if wait_timeout is None:
|
||||
_, status = waitpid(self._pid, 0)
|
||||
else:
|
||||
# gh-146313: A forked child may still hold the pipe's write
|
||||
# end open, preventing the tracker from seeing EOF and
|
||||
# exiting. Poll with WNOHANG to avoid blocking forever.
|
||||
deadline = monotonic() + wait_timeout
|
||||
delay = 0.001
|
||||
while True:
|
||||
result_pid, status = waitpid(self._pid, WNOHANG)
|
||||
if result_pid != 0:
|
||||
break
|
||||
remaining = deadline - monotonic()
|
||||
if remaining <= 0:
|
||||
# The tracker is still running; it will be
|
||||
# reparented to PID 1 (or the nearest subreaper)
|
||||
# when we exit, and reaped there once all pipe
|
||||
# holders release their fd.
|
||||
self._pid = None
|
||||
self._exitcode = None
|
||||
self._waitpid_timed_out = True
|
||||
return
|
||||
delay = min(delay * 2, remaining, 0.1)
|
||||
sleep(delay)
|
||||
except ChildProcessError:
|
||||
self._pid = None
|
||||
self._exitcode = None
|
||||
@@ -312,12 +375,24 @@ class ResourceTracker(object):
|
||||
|
||||
self._ensure_running_and_write(msg)
|
||||
|
||||
# gh-146313: Per-thread flag set by .popen_fork.Popen._launch() just before
|
||||
# os.fork(), telling _after_fork_in_child() to keep the inherited pipe fd so
|
||||
# the child can reuse this tracker (gh-80849). Unset for raw os.fork() calls,
|
||||
# where the child instead closes the fd so the parent's __del__ can reap the
|
||||
# tracker. Using threading.local() keeps multiple threads calling
|
||||
# popen_fork.Popen._launch() at once from clobbering eachothers intent.
|
||||
_fork_intent = threading.local()
|
||||
|
||||
_resource_tracker = ResourceTracker()
|
||||
ensure_running = _resource_tracker.ensure_running
|
||||
register = _resource_tracker.register
|
||||
unregister = _resource_tracker.unregister
|
||||
getfd = _resource_tracker.getfd
|
||||
|
||||
# gh-146313: See _after_fork_in_child docstring.
|
||||
if hasattr(os, 'register_at_fork'):
|
||||
os.register_at_fork(after_in_child=_resource_tracker._after_fork_in_child)
|
||||
|
||||
|
||||
def _decode_message(line):
|
||||
if line.startswith(b'{'):
|
||||
|
||||
2
Lib/multiprocessing/shared_memory.py
vendored
2
Lib/multiprocessing/shared_memory.py
vendored
@@ -539,6 +539,6 @@ class ShareableList:
|
||||
if value == entry:
|
||||
return position
|
||||
else:
|
||||
raise ValueError(f"{value!r} not in this container")
|
||||
raise ValueError("ShareableList.index(x): x not in list")
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
2
Lib/multiprocessing/spawn.py
vendored
2
Lib/multiprocessing/spawn.py
vendored
@@ -184,7 +184,7 @@ def get_preparation_data(name):
|
||||
sys_argv=sys.argv,
|
||||
orig_dir=process.ORIGINAL_DIR,
|
||||
dir=os.getcwd(),
|
||||
start_method=get_start_method(),
|
||||
start_method=get_start_method(allow_none=True),
|
||||
)
|
||||
|
||||
# Figure out whether to initialise main in the subprocess as a module
|
||||
|
||||
31
Lib/multiprocessing/synchronize.py
vendored
31
Lib/multiprocessing/synchronize.py
vendored
@@ -21,22 +21,21 @@ from . import context
|
||||
from . import process
|
||||
from . import util
|
||||
|
||||
# Try to import the mp.synchronize module cleanly, if it fails
|
||||
# raise ImportError for platforms lacking a working sem_open implementation.
|
||||
# See issue 3770
|
||||
# TODO: Do any platforms still lack a functioning sem_open?
|
||||
try:
|
||||
from _multiprocessing import SemLock, sem_unlink
|
||||
except (ImportError):
|
||||
except ImportError:
|
||||
raise ImportError("This platform lacks a functioning sem_open" +
|
||||
" implementation, therefore, the required" +
|
||||
" synchronization primitives needed will not" +
|
||||
" function, see issue 3770.")
|
||||
" implementation. https://github.com/python/cpython/issues/48020.")
|
||||
|
||||
#
|
||||
# Constants
|
||||
#
|
||||
|
||||
RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
|
||||
# These match the enum in Modules/_multiprocessing/semaphore.c
|
||||
RECURSIVE_MUTEX = 0
|
||||
SEMAPHORE = 1
|
||||
|
||||
SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
|
||||
|
||||
#
|
||||
@@ -91,6 +90,9 @@ class SemLock(object):
|
||||
self.acquire = self._semlock.acquire
|
||||
self.release = self._semlock.release
|
||||
|
||||
def locked(self):
|
||||
return self._semlock._is_zero()
|
||||
|
||||
def __enter__(self):
|
||||
return self._semlock.__enter__()
|
||||
|
||||
@@ -133,11 +135,16 @@ class Semaphore(SemLock):
|
||||
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
|
||||
|
||||
def get_value(self):
|
||||
'''Returns current value of Semaphore.
|
||||
|
||||
Raises NotImplementedError on Mac OSX
|
||||
because of broken sem_getvalue().
|
||||
'''
|
||||
return self._semlock._get_value()
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
value = self._semlock._get_value()
|
||||
value = self.get_value()
|
||||
except Exception:
|
||||
value = 'unknown'
|
||||
return '<%s(value=%s)>' % (self.__class__.__name__, value)
|
||||
@@ -153,7 +160,7 @@ class BoundedSemaphore(Semaphore):
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
value = self._semlock._get_value()
|
||||
value = self.get_value()
|
||||
except Exception:
|
||||
value = 'unknown'
|
||||
return '<%s(value=%s, maxvalue=%s)>' % \
|
||||
@@ -245,8 +252,8 @@ class Condition(object):
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
num_waiters = (self._sleeping_count._semlock._get_value() -
|
||||
self._woken_count._semlock._get_value())
|
||||
num_waiters = (self._sleeping_count.get_value() -
|
||||
self._woken_count.get_value())
|
||||
except Exception:
|
||||
num_waiters = 'unknown'
|
||||
return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters)
|
||||
|
||||
14
Lib/multiprocessing/util.py
vendored
14
Lib/multiprocessing/util.py
vendored
@@ -14,12 +14,12 @@ import weakref
|
||||
import atexit
|
||||
import threading # we want threading to install it's
|
||||
# cleanup function before multiprocessing does
|
||||
from subprocess import _args_from_interpreter_flags
|
||||
from subprocess import _args_from_interpreter_flags # noqa: F401
|
||||
|
||||
from . import process
|
||||
|
||||
__all__ = [
|
||||
'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
|
||||
'sub_debug', 'debug', 'info', 'sub_warning', 'warn', 'get_logger',
|
||||
'log_to_stderr', 'get_temp_dir', 'register_after_fork',
|
||||
'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
|
||||
'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
|
||||
@@ -54,7 +54,7 @@ def info(msg, *args):
|
||||
if _logger:
|
||||
_logger.log(INFO, msg, *args, stacklevel=2)
|
||||
|
||||
def _warn(msg, *args):
|
||||
def warn(msg, *args):
|
||||
if _logger:
|
||||
_logger.log(WARNING, msg, *args, stacklevel=2)
|
||||
|
||||
@@ -196,14 +196,14 @@ def _get_base_temp_dir(tempfile):
|
||||
try:
|
||||
base_system_tempdir = tempfile._get_default_tempdir(dirlist)
|
||||
except FileNotFoundError:
|
||||
_warn("Process-wide temporary directory %s will not be usable for "
|
||||
"creating socket files and no usable system-wide temporary "
|
||||
"directory was found in %s", base_tempdir, dirlist)
|
||||
warn("Process-wide temporary directory %s will not be usable for "
|
||||
"creating socket files and no usable system-wide temporary "
|
||||
"directory was found in %s", base_tempdir, dirlist)
|
||||
# At this point, the system-wide temporary directory is not usable
|
||||
# but we may assume that the user-defined one is, even if we will
|
||||
# not be able to write socket files out there.
|
||||
return base_tempdir
|
||||
_warn("Ignoring user-defined temporary directory: %s", base_tempdir)
|
||||
warn("Ignoring user-defined temporary directory: %s", base_tempdir)
|
||||
# at most max(map(len, dirlist)) + 14 + 14 = 36 characters
|
||||
assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX
|
||||
return base_system_tempdir
|
||||
|
||||
12
Lib/pickle.py
vendored
12
Lib/pickle.py
vendored
@@ -904,17 +904,11 @@ class _Pickler:
|
||||
# Write data in-band
|
||||
# XXX The C implementation avoids a copy here
|
||||
buf = m.tobytes()
|
||||
in_memo = id(buf) in self.memo
|
||||
if m.readonly:
|
||||
if in_memo:
|
||||
self._save_bytes_no_memo(buf)
|
||||
else:
|
||||
self.save_bytes(buf)
|
||||
self._save_bytes_no_memo(buf)
|
||||
else:
|
||||
if in_memo:
|
||||
self._save_bytearray_no_memo(buf)
|
||||
else:
|
||||
self.save_bytearray(buf)
|
||||
self._save_bytearray_no_memo(buf)
|
||||
self.memoize(obj)
|
||||
else:
|
||||
# Write data out-of-band
|
||||
self.write(NEXT_BUFFER)
|
||||
|
||||
117
Lib/platform.py
vendored
Executable file → Normal file
117
Lib/platform.py
vendored
Executable file → Normal file
@@ -1,5 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
""" This module tries to retrieve as much platform-identifying data as
|
||||
possible. It makes this information available via function APIs.
|
||||
|
||||
@@ -33,6 +31,7 @@
|
||||
#
|
||||
# <see CVS and SVN checkin messages for history>
|
||||
#
|
||||
# 1.0.9 - added invalidate_caches() function to invalidate cached values
|
||||
# 1.0.8 - changed Windows support to read version from kernel32.dll
|
||||
# 1.0.7 - added DEV_NULL
|
||||
# 1.0.6 - added linux_distribution()
|
||||
@@ -111,7 +110,7 @@ __copyright__ = """
|
||||
|
||||
"""
|
||||
|
||||
__version__ = '1.0.8'
|
||||
__version__ = '1.0.9'
|
||||
|
||||
import collections
|
||||
import os
|
||||
@@ -174,6 +173,11 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384):
|
||||
|
||||
"""
|
||||
if not executable:
|
||||
if sys.platform == "emscripten":
|
||||
# Emscripten's os.confstr reports that it is glibc, so special case
|
||||
# it.
|
||||
ver = ".".join(str(x) for x in sys._emscripten_info.emscripten_version)
|
||||
return ("emscripten", ver)
|
||||
try:
|
||||
ver = os.confstr('CS_GNU_LIBC_VERSION')
|
||||
# parse 'glibc 2.28' as ('glibc', '2.28')
|
||||
@@ -190,22 +194,26 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384):
|
||||
# sys.executable is not set.
|
||||
return lib, version
|
||||
|
||||
libc_search = re.compile(b'(__libc_init)'
|
||||
b'|'
|
||||
b'(GLIBC_([0-9.]+))'
|
||||
b'|'
|
||||
br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII)
|
||||
libc_search = re.compile(br"""
|
||||
(__libc_init)
|
||||
| (GLIBC_([0-9.]+))
|
||||
| (libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)
|
||||
| (musl-([0-9.]+))
|
||||
| ((?:libc\.|ld-)musl(?:-\w+)?.so(?:\.(\d[0-9.]*))?)
|
||||
""",
|
||||
re.ASCII | re.VERBOSE)
|
||||
|
||||
V = _comparable_version
|
||||
# We use os.path.realpath()
|
||||
# here to work around problems with Cygwin not being
|
||||
# able to open symlinks for reading
|
||||
executable = os.path.realpath(executable)
|
||||
ver = None
|
||||
with open(executable, 'rb') as f:
|
||||
binary = f.read(chunksize)
|
||||
pos = 0
|
||||
while pos < len(binary):
|
||||
if b'libc' in binary or b'GLIBC' in binary:
|
||||
if b'libc' in binary or b'GLIBC' in binary or b'musl' in binary:
|
||||
m = libc_search.search(binary, pos)
|
||||
else:
|
||||
m = None
|
||||
@@ -217,26 +225,35 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384):
|
||||
continue
|
||||
if not m:
|
||||
break
|
||||
libcinit, glibc, glibcversion, so, threads, soversion = [
|
||||
s.decode('latin1') if s is not None else s
|
||||
for s in m.groups()]
|
||||
decoded_groups = [s.decode('latin1') if s is not None else s
|
||||
for s in m.groups()]
|
||||
(libcinit, glibc, glibcversion, so, threads, soversion,
|
||||
musl, muslversion, musl_so, musl_sover) = decoded_groups
|
||||
if libcinit and not lib:
|
||||
lib = 'libc'
|
||||
elif glibc:
|
||||
if lib != 'glibc':
|
||||
lib = 'glibc'
|
||||
version = glibcversion
|
||||
elif V(glibcversion) > V(version):
|
||||
version = glibcversion
|
||||
ver = glibcversion
|
||||
elif V(glibcversion) > V(ver):
|
||||
ver = glibcversion
|
||||
elif so:
|
||||
if lib != 'glibc':
|
||||
if lib not in ('glibc', 'musl'):
|
||||
lib = 'libc'
|
||||
if soversion and (not version or V(soversion) > V(version)):
|
||||
version = soversion
|
||||
if threads and version[-len(threads):] != threads:
|
||||
version = version + threads
|
||||
if soversion and (not ver or V(soversion) > V(ver)):
|
||||
ver = soversion
|
||||
if threads and ver[-len(threads):] != threads:
|
||||
ver = ver + threads
|
||||
elif musl:
|
||||
lib = 'musl'
|
||||
if not ver or V(muslversion) > V(ver):
|
||||
ver = muslversion
|
||||
elif musl_so:
|
||||
lib = 'musl'
|
||||
if musl_sover and (not ver or V(musl_sover) > V(ver)):
|
||||
ver = musl_sover
|
||||
pos = m.end()
|
||||
return lib, version
|
||||
return lib, version if ver is None else ver
|
||||
|
||||
def _norm_version(version, build=''):
|
||||
|
||||
@@ -549,7 +566,7 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')):
|
||||
warnings._deprecated('java_ver', remove=(3, 15))
|
||||
# Import the needed APIs
|
||||
try:
|
||||
import java.lang
|
||||
import java.lang # noqa: F401
|
||||
except ImportError:
|
||||
return release, vendor, vminfo, osinfo
|
||||
|
||||
@@ -1192,7 +1209,7 @@ def _sys_version(sys_version=None):
|
||||
# CPython
|
||||
cpython_sys_version_parser = re.compile(
|
||||
r'([\w.+]+)\s*' # "version<space>"
|
||||
r'(?:experimental free-threading build\s+)?' # "free-threading-build<space>"
|
||||
r'(?:free-threading build\s+)?' # "free-threading-build<space>"
|
||||
r'\(#?([^,]+)' # "(#buildno"
|
||||
r'(?:,\s*([\w ]*)' # ", builddate"
|
||||
r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>"
|
||||
@@ -1449,11 +1466,55 @@ def freedesktop_os_release():
|
||||
return _os_release_cache.copy()
|
||||
|
||||
|
||||
def invalidate_caches():
|
||||
"""Invalidate the cached results."""
|
||||
global _uname_cache
|
||||
_uname_cache = None
|
||||
|
||||
global _os_release_cache
|
||||
_os_release_cache = None
|
||||
|
||||
_sys_version_cache.clear()
|
||||
_platform_cache.clear()
|
||||
|
||||
|
||||
### Command line interface
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Default is to print the aliased verbose platform string
|
||||
terse = ('terse' in sys.argv or '--terse' in sys.argv)
|
||||
aliased = (not 'nonaliased' in sys.argv and not '--nonaliased' in sys.argv)
|
||||
def _parse_args(args: list[str] | None):
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(color=True)
|
||||
parser.add_argument("args", nargs="*", choices=["nonaliased", "terse"])
|
||||
parser.add_argument(
|
||||
"--terse",
|
||||
action="store_true",
|
||||
help=(
|
||||
"return only the absolute minimum information needed "
|
||||
"to identify the platform"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--nonaliased",
|
||||
dest="aliased",
|
||||
action="store_false",
|
||||
help=(
|
||||
"disable system/OS name aliasing. If aliasing is enabled, "
|
||||
"some platforms report system names different from "
|
||||
"their common names, e.g. SunOS is reported as Solaris"
|
||||
),
|
||||
)
|
||||
|
||||
return parser.parse_args(args)
|
||||
|
||||
|
||||
def _main(args: list[str] | None = None):
|
||||
args = _parse_args(args)
|
||||
|
||||
terse = args.terse or ("terse" in args.args)
|
||||
aliased = args.aliased and ('nonaliased' not in args.args)
|
||||
|
||||
print(platform(aliased, terse))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
|
||||
6
Lib/plistlib.py
vendored
6
Lib/plistlib.py
vendored
@@ -21,7 +21,7 @@ datetime.datetime objects.
|
||||
|
||||
Generate Plist example:
|
||||
|
||||
import datetime
|
||||
import datetime as dt
|
||||
import plistlib
|
||||
|
||||
pl = dict(
|
||||
@@ -37,7 +37,7 @@ Generate Plist example:
|
||||
),
|
||||
someData = b"<binary gunk>",
|
||||
someMoreData = b"<lots of binary gunk>" * 10,
|
||||
aDate = datetime.datetime.now()
|
||||
aDate = dt.datetime.now()
|
||||
)
|
||||
print(plistlib.dumps(pl).decode())
|
||||
|
||||
@@ -384,7 +384,7 @@ class _PlistWriter(_DumbXMLWriter):
|
||||
self._indent_level -= 1
|
||||
maxlinelength = max(
|
||||
16,
|
||||
76 - len(self.indent.replace(b"\t", b" " * 8) * self._indent_level))
|
||||
76 - len((self.indent * self._indent_level).expandtabs()))
|
||||
|
||||
for line in _encode_base64(data, maxlinelength).split(b"\n"):
|
||||
if line:
|
||||
|
||||
615
Lib/profile.py
vendored
Normal file
615
Lib/profile.py
vendored
Normal file
@@ -0,0 +1,615 @@
|
||||
#
|
||||
# Class for profiling python code. rev 1.0 6/2/94
|
||||
#
|
||||
# Written by James Roskind
|
||||
# Based on prior profile module by Sjoerd Mullender...
|
||||
# which was hacked somewhat by: Guido van Rossum
|
||||
|
||||
"""Class for profiling Python code."""
|
||||
|
||||
# Copyright Disney Enterprises, Inc. All Rights Reserved.
|
||||
# Licensed to PSF under a Contributor Agreement
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
# either express or implied. See the License for the specific language
|
||||
# governing permissions and limitations under the License.
|
||||
|
||||
|
||||
import importlib.machinery
|
||||
import io
|
||||
import sys
|
||||
import time
|
||||
import marshal
|
||||
|
||||
__all__ = ["run", "runctx", "Profile"]
|
||||
|
||||
# Sample timer for use with
|
||||
#i_count = 0
|
||||
#def integer_timer():
|
||||
# global i_count
|
||||
# i_count = i_count + 1
|
||||
# return i_count
|
||||
#itimes = integer_timer # replace with C coded timer returning integers
|
||||
|
||||
class _Utils:
|
||||
"""Support class for utility functions which are shared by
|
||||
profile.py and cProfile.py modules.
|
||||
Not supposed to be used directly.
|
||||
"""
|
||||
|
||||
def __init__(self, profiler):
|
||||
self.profiler = profiler
|
||||
|
||||
def run(self, statement, filename, sort):
|
||||
prof = self.profiler()
|
||||
try:
|
||||
prof.run(statement)
|
||||
except SystemExit:
|
||||
pass
|
||||
finally:
|
||||
self._show(prof, filename, sort)
|
||||
|
||||
def runctx(self, statement, globals, locals, filename, sort):
|
||||
prof = self.profiler()
|
||||
try:
|
||||
prof.runctx(statement, globals, locals)
|
||||
except SystemExit:
|
||||
pass
|
||||
finally:
|
||||
self._show(prof, filename, sort)
|
||||
|
||||
def _show(self, prof, filename, sort):
|
||||
if filename is not None:
|
||||
prof.dump_stats(filename)
|
||||
else:
|
||||
prof.print_stats(sort)
|
||||
|
||||
|
||||
#**************************************************************************
|
||||
# The following are the static member functions for the profiler class
|
||||
# Note that an instance of Profile() is *not* needed to call them.
|
||||
#**************************************************************************
|
||||
|
||||
def run(statement, filename=None, sort=-1):
|
||||
"""Run statement under profiler optionally saving results in filename
|
||||
|
||||
This function takes a single argument that can be passed to the
|
||||
"exec" statement, and an optional file name. In all cases this
|
||||
routine attempts to "exec" its first argument and gather profiling
|
||||
statistics from the execution. If no file name is present, then this
|
||||
function automatically prints a simple profiling report, sorted by the
|
||||
standard name string (file/line/function-name) that is presented in
|
||||
each line.
|
||||
"""
|
||||
return _Utils(Profile).run(statement, filename, sort)
|
||||
|
||||
def runctx(statement, globals, locals, filename=None, sort=-1):
|
||||
"""Run statement under profiler, supplying your own globals and locals,
|
||||
optionally saving results in filename.
|
||||
|
||||
statement and filename have the same semantics as profile.run
|
||||
"""
|
||||
return _Utils(Profile).runctx(statement, globals, locals, filename, sort)
|
||||
|
||||
|
||||
class Profile:
|
||||
"""Profiler class.
|
||||
|
||||
self.cur is always a tuple. Each such tuple corresponds to a stack
|
||||
frame that is currently active (self.cur[-2]). The following are the
|
||||
definitions of its members. We use this external "parallel stack" to
|
||||
avoid contaminating the program that we are profiling. (old profiler
|
||||
used to write into the frames local dictionary!!) Derived classes
|
||||
can change the definition of some entries, as long as they leave
|
||||
[-2:] intact (frame and previous tuple). In case an internal error is
|
||||
detected, the -3 element is used as the function name.
|
||||
|
||||
[ 0] = Time that needs to be charged to the parent frame's function.
|
||||
It is used so that a function call will not have to access the
|
||||
timing data for the parent frame.
|
||||
[ 1] = Total time spent in this frame's function, excluding time in
|
||||
subfunctions (this latter is tallied in cur[2]).
|
||||
[ 2] = Total time spent in subfunctions, excluding time executing the
|
||||
frame's function (this latter is tallied in cur[1]).
|
||||
[-3] = Name of the function that corresponds to this frame.
|
||||
[-2] = Actual frame that we correspond to (used to sync exception handling).
|
||||
[-1] = Our parent 6-tuple (corresponds to frame.f_back).
|
||||
|
||||
Timing data for each function is stored as a 5-tuple in the dictionary
|
||||
self.timings[]. The index is always the name stored in self.cur[-3].
|
||||
The following are the definitions of the members:
|
||||
|
||||
[0] = The number of times this function was called, not counting direct
|
||||
or indirect recursion,
|
||||
[1] = Number of times this function appears on the stack, minus one
|
||||
[2] = Total time spent internal to this function
|
||||
[3] = Cumulative time that this function was present on the stack. In
|
||||
non-recursive functions, this is the total execution time from start
|
||||
to finish of each invocation of a function, including time spent in
|
||||
all subfunctions.
|
||||
[4] = A dictionary indicating for each function name, the number of times
|
||||
it was called by us.
|
||||
"""
|
||||
|
||||
bias = 0 # calibration constant
|
||||
|
||||
def __init__(self, timer=None, bias=None):
|
||||
self.timings = {}
|
||||
self.cur = None
|
||||
self.cmd = ""
|
||||
self.c_func_name = ""
|
||||
|
||||
if bias is None:
|
||||
bias = self.bias
|
||||
self.bias = bias # Materialize in local dict for lookup speed.
|
||||
|
||||
if not timer:
|
||||
self.timer = self.get_time = time.process_time
|
||||
self.dispatcher = self.trace_dispatch_i
|
||||
else:
|
||||
self.timer = timer
|
||||
t = self.timer() # test out timer function
|
||||
try:
|
||||
length = len(t)
|
||||
except TypeError:
|
||||
self.get_time = timer
|
||||
self.dispatcher = self.trace_dispatch_i
|
||||
else:
|
||||
if length == 2:
|
||||
self.dispatcher = self.trace_dispatch
|
||||
else:
|
||||
self.dispatcher = self.trace_dispatch_l
|
||||
# This get_time() implementation needs to be defined
|
||||
# here to capture the passed-in timer in the parameter
|
||||
# list (for performance). Note that we can't assume
|
||||
# the timer() result contains two values in all
|
||||
# cases.
|
||||
def get_time_timer(timer=timer, sum=sum):
|
||||
return sum(timer())
|
||||
self.get_time = get_time_timer
|
||||
self.t = self.get_time()
|
||||
self.simulate_call('profiler')
|
||||
|
||||
# Heavily optimized dispatch routine for time.process_time() timer
|
||||
|
||||
def trace_dispatch(self, frame, event, arg):
|
||||
timer = self.timer
|
||||
t = timer()
|
||||
t = t[0] + t[1] - self.t - self.bias
|
||||
|
||||
if event == "c_call":
|
||||
self.c_func_name = arg.__name__
|
||||
|
||||
if self.dispatch[event](self, frame,t):
|
||||
t = timer()
|
||||
self.t = t[0] + t[1]
|
||||
else:
|
||||
r = timer()
|
||||
self.t = r[0] + r[1] - t # put back unrecorded delta
|
||||
|
||||
# Dispatch routine for best timer program (return = scalar, fastest if
|
||||
# an integer but float works too -- and time.process_time() relies on that).
|
||||
|
||||
def trace_dispatch_i(self, frame, event, arg):
|
||||
timer = self.timer
|
||||
t = timer() - self.t - self.bias
|
||||
|
||||
if event == "c_call":
|
||||
self.c_func_name = arg.__name__
|
||||
|
||||
if self.dispatch[event](self, frame, t):
|
||||
self.t = timer()
|
||||
else:
|
||||
self.t = timer() - t # put back unrecorded delta
|
||||
|
||||
# Dispatch routine for macintosh (timer returns time in ticks of
|
||||
# 1/60th second)
|
||||
|
||||
def trace_dispatch_mac(self, frame, event, arg):
|
||||
timer = self.timer
|
||||
t = timer()/60.0 - self.t - self.bias
|
||||
|
||||
if event == "c_call":
|
||||
self.c_func_name = arg.__name__
|
||||
|
||||
if self.dispatch[event](self, frame, t):
|
||||
self.t = timer()/60.0
|
||||
else:
|
||||
self.t = timer()/60.0 - t # put back unrecorded delta
|
||||
|
||||
# SLOW generic dispatch routine for timer returning lists of numbers
|
||||
|
||||
def trace_dispatch_l(self, frame, event, arg):
|
||||
get_time = self.get_time
|
||||
t = get_time() - self.t - self.bias
|
||||
|
||||
if event == "c_call":
|
||||
self.c_func_name = arg.__name__
|
||||
|
||||
if self.dispatch[event](self, frame, t):
|
||||
self.t = get_time()
|
||||
else:
|
||||
self.t = get_time() - t # put back unrecorded delta
|
||||
|
||||
# In the event handlers, the first 3 elements of self.cur are unpacked
|
||||
# into vrbls w/ 3-letter names. The last two characters are meant to be
|
||||
# mnemonic:
|
||||
# _pt self.cur[0] "parent time" time to be charged to parent frame
|
||||
# _it self.cur[1] "internal time" time spent directly in the function
|
||||
# _et self.cur[2] "external time" time spent in subfunctions
|
||||
|
||||
def trace_dispatch_exception(self, frame, t):
|
||||
rpt, rit, ret, rfn, rframe, rcur = self.cur
|
||||
if (rframe is not frame) and rcur:
|
||||
return self.trace_dispatch_return(rframe, t)
|
||||
self.cur = rpt, rit+t, ret, rfn, rframe, rcur
|
||||
return 1
|
||||
|
||||
|
||||
def trace_dispatch_call(self, frame, t):
|
||||
if self.cur and frame.f_back is not self.cur[-2]:
|
||||
rpt, rit, ret, rfn, rframe, rcur = self.cur
|
||||
if not isinstance(rframe, Profile.fake_frame):
|
||||
assert rframe.f_back is frame.f_back, ("Bad call", rfn,
|
||||
rframe, rframe.f_back,
|
||||
frame, frame.f_back)
|
||||
self.trace_dispatch_return(rframe, 0)
|
||||
assert (self.cur is None or \
|
||||
frame.f_back is self.cur[-2]), ("Bad call",
|
||||
self.cur[-3])
|
||||
fcode = frame.f_code
|
||||
fn = (fcode.co_filename, fcode.co_firstlineno, fcode.co_name)
|
||||
self.cur = (t, 0, 0, fn, frame, self.cur)
|
||||
timings = self.timings
|
||||
if fn in timings:
|
||||
cc, ns, tt, ct, callers = timings[fn]
|
||||
timings[fn] = cc, ns + 1, tt, ct, callers
|
||||
else:
|
||||
timings[fn] = 0, 0, 0, 0, {}
|
||||
return 1
|
||||
|
||||
def trace_dispatch_c_call (self, frame, t):
|
||||
fn = ("", 0, self.c_func_name)
|
||||
self.cur = (t, 0, 0, fn, frame, self.cur)
|
||||
timings = self.timings
|
||||
if fn in timings:
|
||||
cc, ns, tt, ct, callers = timings[fn]
|
||||
timings[fn] = cc, ns+1, tt, ct, callers
|
||||
else:
|
||||
timings[fn] = 0, 0, 0, 0, {}
|
||||
return 1
|
||||
|
||||
def trace_dispatch_return(self, frame, t):
|
||||
if frame is not self.cur[-2]:
|
||||
assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3])
|
||||
self.trace_dispatch_return(self.cur[-2], 0)
|
||||
|
||||
# Prefix "r" means part of the Returning or exiting frame.
|
||||
# Prefix "p" means part of the Previous or Parent or older frame.
|
||||
|
||||
rpt, rit, ret, rfn, frame, rcur = self.cur
|
||||
rit = rit + t
|
||||
frame_total = rit + ret
|
||||
|
||||
ppt, pit, pet, pfn, pframe, pcur = rcur
|
||||
self.cur = ppt, pit + rpt, pet + frame_total, pfn, pframe, pcur
|
||||
|
||||
timings = self.timings
|
||||
cc, ns, tt, ct, callers = timings[rfn]
|
||||
if not ns:
|
||||
# This is the only occurrence of the function on the stack.
|
||||
# Else this is a (directly or indirectly) recursive call, and
|
||||
# its cumulative time will get updated when the topmost call to
|
||||
# it returns.
|
||||
ct = ct + frame_total
|
||||
cc = cc + 1
|
||||
|
||||
if pfn in callers:
|
||||
callers[pfn] = callers[pfn] + 1 # hack: gather more
|
||||
# stats such as the amount of time added to ct courtesy
|
||||
# of this specific call, and the contribution to cc
|
||||
# courtesy of this call.
|
||||
else:
|
||||
callers[pfn] = 1
|
||||
|
||||
timings[rfn] = cc, ns - 1, tt + rit, ct, callers
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
dispatch = {
|
||||
"call": trace_dispatch_call,
|
||||
"exception": trace_dispatch_exception,
|
||||
"return": trace_dispatch_return,
|
||||
"c_call": trace_dispatch_c_call,
|
||||
"c_exception": trace_dispatch_return, # the C function returned
|
||||
"c_return": trace_dispatch_return,
|
||||
}
|
||||
|
||||
|
||||
# The next few functions play with self.cmd. By carefully preloading
|
||||
# our parallel stack, we can force the profiled result to include
|
||||
# an arbitrary string as the name of the calling function.
|
||||
# We use self.cmd as that string, and the resulting stats look
|
||||
# very nice :-).
|
||||
|
||||
def set_cmd(self, cmd):
|
||||
if self.cur[-1]: return # already set
|
||||
self.cmd = cmd
|
||||
self.simulate_call(cmd)
|
||||
|
||||
class fake_code:
|
||||
def __init__(self, filename, line, name):
|
||||
self.co_filename = filename
|
||||
self.co_line = line
|
||||
self.co_name = name
|
||||
self.co_firstlineno = 0
|
||||
|
||||
def __repr__(self):
|
||||
return repr((self.co_filename, self.co_line, self.co_name))
|
||||
|
||||
class fake_frame:
|
||||
def __init__(self, code, prior):
|
||||
self.f_code = code
|
||||
self.f_back = prior
|
||||
|
||||
def simulate_call(self, name):
|
||||
code = self.fake_code('profile', 0, name)
|
||||
if self.cur:
|
||||
pframe = self.cur[-2]
|
||||
else:
|
||||
pframe = None
|
||||
frame = self.fake_frame(code, pframe)
|
||||
self.dispatch['call'](self, frame, 0)
|
||||
|
||||
# collect stats from pending stack, including getting final
|
||||
# timings for self.cmd frame.
|
||||
|
||||
def simulate_cmd_complete(self):
|
||||
get_time = self.get_time
|
||||
t = get_time() - self.t
|
||||
while self.cur[-1]:
|
||||
# We *can* cause assertion errors here if
|
||||
# dispatch_trace_return checks for a frame match!
|
||||
self.dispatch['return'](self, self.cur[-2], t)
|
||||
t = 0
|
||||
self.t = get_time() - t
|
||||
|
||||
|
||||
def print_stats(self, sort=-1):
|
||||
import pstats
|
||||
if not isinstance(sort, tuple):
|
||||
sort = (sort,)
|
||||
pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats()
|
||||
|
||||
def dump_stats(self, file):
|
||||
with open(file, 'wb') as f:
|
||||
self.create_stats()
|
||||
marshal.dump(self.stats, f)
|
||||
|
||||
def create_stats(self):
|
||||
self.simulate_cmd_complete()
|
||||
self.snapshot_stats()
|
||||
|
||||
def snapshot_stats(self):
|
||||
self.stats = {}
|
||||
for func, (cc, ns, tt, ct, callers) in self.timings.items():
|
||||
callers = callers.copy()
|
||||
nc = 0
|
||||
for callcnt in callers.values():
|
||||
nc += callcnt
|
||||
self.stats[func] = cc, nc, tt, ct, callers
|
||||
|
||||
|
||||
# The following two methods can be called by clients to use
|
||||
# a profiler to profile a statement, given as a string.
|
||||
|
||||
def run(self, cmd):
|
||||
import __main__
|
||||
dict = __main__.__dict__
|
||||
return self.runctx(cmd, dict, dict)
|
||||
|
||||
def runctx(self, cmd, globals, locals):
|
||||
self.set_cmd(cmd)
|
||||
sys.setprofile(self.dispatcher)
|
||||
try:
|
||||
exec(cmd, globals, locals)
|
||||
finally:
|
||||
sys.setprofile(None)
|
||||
return self
|
||||
|
||||
# This method is more useful to profile a single function call.
|
||||
def runcall(self, func, /, *args, **kw):
|
||||
self.set_cmd(repr(func))
|
||||
sys.setprofile(self.dispatcher)
|
||||
try:
|
||||
return func(*args, **kw)
|
||||
finally:
|
||||
sys.setprofile(None)
|
||||
|
||||
|
||||
#******************************************************************
|
||||
# The following calculates the overhead for using a profiler. The
|
||||
# problem is that it takes a fair amount of time for the profiler
|
||||
# to stop the stopwatch (from the time it receives an event).
|
||||
# Similarly, there is a delay from the time that the profiler
|
||||
# re-starts the stopwatch before the user's code really gets to
|
||||
# continue. The following code tries to measure the difference on
|
||||
# a per-event basis.
|
||||
#
|
||||
# Note that this difference is only significant if there are a lot of
|
||||
# events, and relatively little user code per event. For example,
|
||||
# code with small functions will typically benefit from having the
|
||||
# profiler calibrated for the current platform. This *could* be
|
||||
# done on the fly during init() time, but it is not worth the
|
||||
# effort. Also note that if too large a value specified, then
|
||||
# execution time on some functions will actually appear as a
|
||||
# negative number. It is *normal* for some functions (with very
|
||||
# low call counts) to have such negative stats, even if the
|
||||
# calibration figure is "correct."
|
||||
#
|
||||
# One alternative to profile-time calibration adjustments (i.e.,
|
||||
# adding in the magic little delta during each event) is to track
|
||||
# more carefully the number of events (and cumulatively, the number
|
||||
# of events during sub functions) that are seen. If this were
|
||||
# done, then the arithmetic could be done after the fact (i.e., at
|
||||
# display time). Currently, we track only call/return events.
|
||||
# These values can be deduced by examining the callees and callers
|
||||
# vectors for each functions. Hence we *can* almost correct the
|
||||
# internal time figure at print time (note that we currently don't
|
||||
# track exception event processing counts). Unfortunately, there
|
||||
# is currently no similar information for cumulative sub-function
|
||||
# time. It would not be hard to "get all this info" at profiler
|
||||
# time. Specifically, we would have to extend the tuples to keep
|
||||
# counts of this in each frame, and then extend the defs of timing
|
||||
# tuples to include the significant two figures. I'm a bit fearful
|
||||
# that this additional feature will slow the heavily optimized
|
||||
# event/time ratio (i.e., the profiler would run slower, fur a very
|
||||
# low "value added" feature.)
|
||||
#**************************************************************
|
||||
|
||||
def calibrate(self, m, verbose=0):
|
||||
if self.__class__ is not Profile:
|
||||
raise TypeError("Subclasses must override .calibrate().")
|
||||
|
||||
saved_bias = self.bias
|
||||
self.bias = 0
|
||||
try:
|
||||
return self._calibrate_inner(m, verbose)
|
||||
finally:
|
||||
self.bias = saved_bias
|
||||
|
||||
def _calibrate_inner(self, m, verbose):
|
||||
get_time = self.get_time
|
||||
|
||||
# Set up a test case to be run with and without profiling. Include
|
||||
# lots of calls, because we're trying to quantify stopwatch overhead.
|
||||
# Do not raise any exceptions, though, because we want to know
|
||||
# exactly how many profile events are generated (one call event, +
|
||||
# one return event, per Python-level call).
|
||||
|
||||
def f1(n):
|
||||
for i in range(n):
|
||||
x = 1
|
||||
|
||||
def f(m, f1=f1):
|
||||
for i in range(m):
|
||||
f1(100)
|
||||
|
||||
f(m) # warm up the cache
|
||||
|
||||
# elapsed_noprofile <- time f(m) takes without profiling.
|
||||
t0 = get_time()
|
||||
f(m)
|
||||
t1 = get_time()
|
||||
elapsed_noprofile = t1 - t0
|
||||
if verbose:
|
||||
print("elapsed time without profiling =", elapsed_noprofile)
|
||||
|
||||
# elapsed_profile <- time f(m) takes with profiling. The difference
|
||||
# is profiling overhead, only some of which the profiler subtracts
|
||||
# out on its own.
|
||||
p = Profile()
|
||||
t0 = get_time()
|
||||
p.runctx('f(m)', globals(), locals())
|
||||
t1 = get_time()
|
||||
elapsed_profile = t1 - t0
|
||||
if verbose:
|
||||
print("elapsed time with profiling =", elapsed_profile)
|
||||
|
||||
# reported_time <- "CPU seconds" the profiler charged to f and f1.
|
||||
total_calls = 0.0
|
||||
reported_time = 0.0
|
||||
for (filename, line, funcname), (cc, ns, tt, ct, callers) in \
|
||||
p.timings.items():
|
||||
if funcname in ("f", "f1"):
|
||||
total_calls += cc
|
||||
reported_time += tt
|
||||
|
||||
if verbose:
|
||||
print("'CPU seconds' profiler reported =", reported_time)
|
||||
print("total # calls =", total_calls)
|
||||
if total_calls != m + 1:
|
||||
raise ValueError("internal error: total calls = %d" % total_calls)
|
||||
|
||||
# reported_time - elapsed_noprofile = overhead the profiler wasn't
|
||||
# able to measure. Divide by twice the number of calls (since there
|
||||
# are two profiler events per call in this test) to get the hidden
|
||||
# overhead per event.
|
||||
mean = (reported_time - elapsed_noprofile) / 2.0 / total_calls
|
||||
if verbose:
|
||||
print("mean stopwatch overhead per profile event =", mean)
|
||||
return mean
|
||||
|
||||
#****************************************************************************
|
||||
|
||||
def main():
|
||||
import os
|
||||
from optparse import OptionParser
|
||||
|
||||
usage = "profile.py [-o output_file_path] [-s sort] [-m module | scriptfile] [arg] ..."
|
||||
parser = OptionParser(usage=usage)
|
||||
parser.allow_interspersed_args = False
|
||||
parser.add_option('-o', '--outfile', dest="outfile",
|
||||
help="Save stats to <outfile>", default=None)
|
||||
parser.add_option('-m', dest="module", action="store_true",
|
||||
help="Profile a library module.", default=False)
|
||||
parser.add_option('-s', '--sort', dest="sort",
|
||||
help="Sort order when printing to stdout, based on pstats.Stats class",
|
||||
default=-1)
|
||||
|
||||
if not sys.argv[1:]:
|
||||
parser.print_usage()
|
||||
sys.exit(2)
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
sys.argv[:] = args
|
||||
|
||||
# The script that we're profiling may chdir, so capture the absolute path
|
||||
# to the output file at startup.
|
||||
if options.outfile is not None:
|
||||
options.outfile = os.path.abspath(options.outfile)
|
||||
|
||||
if len(args) > 0:
|
||||
if options.module:
|
||||
import runpy
|
||||
code = "run_module(modname, run_name='__main__')"
|
||||
globs = {
|
||||
'run_module': runpy.run_module,
|
||||
'modname': args[0]
|
||||
}
|
||||
else:
|
||||
progname = args[0]
|
||||
sys.path.insert(0, os.path.dirname(progname))
|
||||
with io.open_code(progname) as fp:
|
||||
code = compile(fp.read(), progname, 'exec')
|
||||
spec = importlib.machinery.ModuleSpec(name='__main__', loader=None,
|
||||
origin=progname)
|
||||
globs = {
|
||||
'__spec__': spec,
|
||||
'__file__': spec.origin,
|
||||
'__name__': spec.name,
|
||||
'__package__': None,
|
||||
'__cached__': None,
|
||||
}
|
||||
try:
|
||||
runctx(code, globs, None, options.outfile, options.sort)
|
||||
except BrokenPipeError as exc:
|
||||
# Prevent "Exception ignored" during interpreter shutdown.
|
||||
sys.stdout = None
|
||||
sys.exit(exc.errno)
|
||||
else:
|
||||
parser.print_usage()
|
||||
return parser
|
||||
|
||||
# When invoked as main program, invoke the profiler on a script
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
777
Lib/pstats.py
vendored
Normal file
777
Lib/pstats.py
vendored
Normal file
@@ -0,0 +1,777 @@
|
||||
"""Class for printing reports on profiled python code."""
|
||||
|
||||
# Written by James Roskind
|
||||
# Based on prior profile module by Sjoerd Mullender...
|
||||
# which was hacked somewhat by: Guido van Rossum
|
||||
|
||||
# Copyright Disney Enterprises, Inc. All Rights Reserved.
|
||||
# Licensed to PSF under a Contributor Agreement
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
# either express or implied. See the License for the specific language
|
||||
# governing permissions and limitations under the License.
|
||||
|
||||
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import marshal
|
||||
import re
|
||||
|
||||
from enum import StrEnum, _simple_enum
|
||||
from functools import cmp_to_key
|
||||
from dataclasses import dataclass
|
||||
|
||||
__all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"]
|
||||
|
||||
@_simple_enum(StrEnum)
|
||||
class SortKey:
|
||||
CALLS = 'calls', 'ncalls'
|
||||
CUMULATIVE = 'cumulative', 'cumtime'
|
||||
FILENAME = 'filename', 'module'
|
||||
LINE = 'line'
|
||||
NAME = 'name'
|
||||
NFL = 'nfl'
|
||||
PCALLS = 'pcalls'
|
||||
STDNAME = 'stdname'
|
||||
TIME = 'time', 'tottime'
|
||||
|
||||
def __new__(cls, *values):
|
||||
value = values[0]
|
||||
obj = str.__new__(cls, value)
|
||||
obj._value_ = value
|
||||
for other_value in values[1:]:
|
||||
cls._value2member_map_[other_value] = obj
|
||||
obj._all_values = values
|
||||
return obj
|
||||
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
class FunctionProfile:
|
||||
ncalls: str
|
||||
tottime: float
|
||||
percall_tottime: float
|
||||
cumtime: float
|
||||
percall_cumtime: float
|
||||
file_name: str
|
||||
line_number: int
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
class StatsProfile:
|
||||
'''Class for keeping track of an item in inventory.'''
|
||||
total_tt: float
|
||||
func_profiles: dict[str, FunctionProfile]
|
||||
|
||||
class Stats:
|
||||
"""This class is used for creating reports from data generated by the
|
||||
Profile class. It is a "friend" of that class, and imports data either
|
||||
by direct access to members of Profile class, or by reading in a dictionary
|
||||
that was emitted (via marshal) from the Profile class.
|
||||
|
||||
The big change from the previous Profiler (in terms of raw functionality)
|
||||
is that an "add()" method has been provided to combine Stats from
|
||||
several distinct profile runs. Both the constructor and the add()
|
||||
method now take arbitrarily many file names as arguments.
|
||||
|
||||
All the print methods now take an argument that indicates how many lines
|
||||
to print. If the arg is a floating-point number between 0 and 1.0, then
|
||||
it is taken as a decimal percentage of the available lines to be printed
|
||||
(e.g., .1 means print 10% of all available lines). If it is an integer,
|
||||
it is taken to mean the number of lines of data that you wish to have
|
||||
printed.
|
||||
|
||||
The sort_stats() method now processes some additional options (i.e., in
|
||||
addition to the old -1, 0, 1, or 2 that are respectively interpreted as
|
||||
'stdname', 'calls', 'time', and 'cumulative'). It takes either an
|
||||
arbitrary number of quoted strings or SortKey enum to select the sort
|
||||
order.
|
||||
|
||||
For example sort_stats('time', 'name') or sort_stats(SortKey.TIME,
|
||||
SortKey.NAME) sorts on the major key of 'internal function time', and on
|
||||
the minor key of 'the name of the function'. Look at the two tables in
|
||||
sort_stats() and get_sort_arg_defs(self) for more examples.
|
||||
|
||||
All methods return self, so you can string together commands like:
|
||||
Stats('foo', 'goo').strip_dirs().sort_stats('calls').\
|
||||
print_stats(5).print_callers(5)
|
||||
"""
|
||||
|
||||
def __init__(self, *args, stream=None):
|
||||
self.stream = stream or sys.stdout
|
||||
if not len(args):
|
||||
arg = None
|
||||
else:
|
||||
arg = args[0]
|
||||
args = args[1:]
|
||||
self.init(arg)
|
||||
self.add(*args)
|
||||
|
||||
def init(self, arg):
|
||||
self.all_callees = None # calc only if needed
|
||||
self.files = []
|
||||
self.fcn_list = None
|
||||
self.total_tt = 0
|
||||
self.total_calls = 0
|
||||
self.prim_calls = 0
|
||||
self.max_name_len = 0
|
||||
self.top_level = set()
|
||||
self.stats = {}
|
||||
self.sort_arg_dict = {}
|
||||
self.load_stats(arg)
|
||||
try:
|
||||
self.get_top_level_stats()
|
||||
except Exception:
|
||||
print("Invalid timing data %s" %
|
||||
(self.files[-1] if self.files else ''), file=self.stream)
|
||||
raise
|
||||
|
||||
def load_stats(self, arg):
|
||||
if arg is None:
|
||||
self.stats = {}
|
||||
return
|
||||
elif isinstance(arg, str):
|
||||
with open(arg, 'rb') as f:
|
||||
self.stats = marshal.load(f)
|
||||
try:
|
||||
file_stats = os.stat(arg)
|
||||
arg = time.ctime(file_stats.st_mtime) + " " + arg
|
||||
except: # in case this is not unix
|
||||
pass
|
||||
self.files = [arg]
|
||||
elif hasattr(arg, 'create_stats'):
|
||||
arg.create_stats()
|
||||
self.stats = arg.stats
|
||||
arg.stats = {}
|
||||
if not self.stats:
|
||||
raise TypeError("Cannot create or construct a %r object from %r"
|
||||
% (self.__class__, arg))
|
||||
return
|
||||
|
||||
def get_top_level_stats(self):
|
||||
for func, (cc, nc, tt, ct, callers) in self.stats.items():
|
||||
self.total_calls += nc
|
||||
self.prim_calls += cc
|
||||
self.total_tt += tt
|
||||
if ("jprofile", 0, "profiler") in callers:
|
||||
self.top_level.add(func)
|
||||
if len(func_std_string(func)) > self.max_name_len:
|
||||
self.max_name_len = len(func_std_string(func))
|
||||
|
||||
def add(self, *arg_list):
|
||||
if not arg_list:
|
||||
return self
|
||||
for item in reversed(arg_list):
|
||||
if type(self) != type(item):
|
||||
item = Stats(item)
|
||||
self.files += item.files
|
||||
self.total_calls += item.total_calls
|
||||
self.prim_calls += item.prim_calls
|
||||
self.total_tt += item.total_tt
|
||||
for func in item.top_level:
|
||||
self.top_level.add(func)
|
||||
|
||||
if self.max_name_len < item.max_name_len:
|
||||
self.max_name_len = item.max_name_len
|
||||
|
||||
self.fcn_list = None
|
||||
|
||||
for func, stat in item.stats.items():
|
||||
if func in self.stats:
|
||||
old_func_stat = self.stats[func]
|
||||
else:
|
||||
old_func_stat = (0, 0, 0, 0, {},)
|
||||
self.stats[func] = add_func_stats(old_func_stat, stat)
|
||||
return self
|
||||
|
||||
def dump_stats(self, filename):
|
||||
"""Write the profile data to a file we know how to load back."""
|
||||
with open(filename, 'wb') as f:
|
||||
marshal.dump(self.stats, f)
|
||||
|
||||
# list the tuple indices and directions for sorting,
|
||||
# along with some printable description
|
||||
sort_arg_dict_default = {
|
||||
"calls" : (((1,-1), ), "call count"),
|
||||
"ncalls" : (((1,-1), ), "call count"),
|
||||
"cumtime" : (((3,-1), ), "cumulative time"),
|
||||
"cumulative": (((3,-1), ), "cumulative time"),
|
||||
"filename" : (((4, 1), ), "file name"),
|
||||
"line" : (((5, 1), ), "line number"),
|
||||
"module" : (((4, 1), ), "file name"),
|
||||
"name" : (((6, 1), ), "function name"),
|
||||
"nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"),
|
||||
"pcalls" : (((0,-1), ), "primitive call count"),
|
||||
"stdname" : (((7, 1), ), "standard name"),
|
||||
"time" : (((2,-1), ), "internal time"),
|
||||
"tottime" : (((2,-1), ), "internal time"),
|
||||
}
|
||||
|
||||
def get_sort_arg_defs(self):
|
||||
"""Expand all abbreviations that are unique."""
|
||||
if not self.sort_arg_dict:
|
||||
self.sort_arg_dict = dict = {}
|
||||
bad_list = {}
|
||||
for word, tup in self.sort_arg_dict_default.items():
|
||||
fragment = word
|
||||
while fragment:
|
||||
if fragment in dict:
|
||||
bad_list[fragment] = 0
|
||||
break
|
||||
dict[fragment] = tup
|
||||
fragment = fragment[:-1]
|
||||
for word in bad_list:
|
||||
del dict[word]
|
||||
return self.sort_arg_dict
|
||||
|
||||
def sort_stats(self, *field):
|
||||
if not field:
|
||||
self.fcn_list = 0
|
||||
return self
|
||||
if len(field) == 1 and isinstance(field[0], int):
|
||||
# Be compatible with old profiler
|
||||
field = [ {-1: "stdname",
|
||||
0: "calls",
|
||||
1: "time",
|
||||
2: "cumulative"}[field[0]] ]
|
||||
elif len(field) >= 2:
|
||||
for arg in field[1:]:
|
||||
if type(arg) != type(field[0]):
|
||||
raise TypeError("Can't have mixed argument type")
|
||||
|
||||
sort_arg_defs = self.get_sort_arg_defs()
|
||||
|
||||
sort_tuple = ()
|
||||
self.sort_type = ""
|
||||
connector = ""
|
||||
for word in field:
|
||||
if isinstance(word, SortKey):
|
||||
word = word.value
|
||||
sort_tuple = sort_tuple + sort_arg_defs[word][0]
|
||||
self.sort_type += connector + sort_arg_defs[word][1]
|
||||
connector = ", "
|
||||
|
||||
stats_list = []
|
||||
for func, (cc, nc, tt, ct, callers) in self.stats.items():
|
||||
stats_list.append((cc, nc, tt, ct) + func +
|
||||
(func_std_string(func), func))
|
||||
|
||||
stats_list.sort(key=cmp_to_key(TupleComp(sort_tuple).compare))
|
||||
|
||||
self.fcn_list = fcn_list = []
|
||||
for tuple in stats_list:
|
||||
fcn_list.append(tuple[-1])
|
||||
return self
|
||||
|
||||
def reverse_order(self):
|
||||
if self.fcn_list:
|
||||
self.fcn_list.reverse()
|
||||
return self
|
||||
|
||||
def strip_dirs(self):
|
||||
oldstats = self.stats
|
||||
self.stats = newstats = {}
|
||||
max_name_len = 0
|
||||
for func, (cc, nc, tt, ct, callers) in oldstats.items():
|
||||
newfunc = func_strip_path(func)
|
||||
if len(func_std_string(newfunc)) > max_name_len:
|
||||
max_name_len = len(func_std_string(newfunc))
|
||||
newcallers = {}
|
||||
for func2, caller in callers.items():
|
||||
newcallers[func_strip_path(func2)] = caller
|
||||
|
||||
if newfunc in newstats:
|
||||
newstats[newfunc] = add_func_stats(
|
||||
newstats[newfunc],
|
||||
(cc, nc, tt, ct, newcallers))
|
||||
else:
|
||||
newstats[newfunc] = (cc, nc, tt, ct, newcallers)
|
||||
old_top = self.top_level
|
||||
self.top_level = new_top = set()
|
||||
for func in old_top:
|
||||
new_top.add(func_strip_path(func))
|
||||
|
||||
self.max_name_len = max_name_len
|
||||
|
||||
self.fcn_list = None
|
||||
self.all_callees = None
|
||||
return self
|
||||
|
||||
def calc_callees(self):
|
||||
if self.all_callees:
|
||||
return
|
||||
self.all_callees = all_callees = {}
|
||||
for func, (cc, nc, tt, ct, callers) in self.stats.items():
|
||||
if not func in all_callees:
|
||||
all_callees[func] = {}
|
||||
for func2, caller in callers.items():
|
||||
if not func2 in all_callees:
|
||||
all_callees[func2] = {}
|
||||
all_callees[func2][func] = caller
|
||||
return
|
||||
|
||||
#******************************************************************
|
||||
# The following functions support actual printing of reports
|
||||
#******************************************************************
|
||||
|
||||
# Optional "amount" is either a line count, or a percentage of lines.
|
||||
|
||||
def eval_print_amount(self, sel, list, msg):
|
||||
new_list = list
|
||||
if isinstance(sel, str):
|
||||
try:
|
||||
rex = re.compile(sel)
|
||||
except re.PatternError:
|
||||
msg += " <Invalid regular expression %r>\n" % sel
|
||||
return new_list, msg
|
||||
new_list = []
|
||||
for func in list:
|
||||
if rex.search(func_std_string(func)):
|
||||
new_list.append(func)
|
||||
else:
|
||||
count = len(list)
|
||||
if isinstance(sel, float) and 0.0 <= sel < 1.0:
|
||||
count = int(count * sel + .5)
|
||||
new_list = list[:count]
|
||||
elif isinstance(sel, int) and 0 <= sel < count:
|
||||
count = sel
|
||||
new_list = list[:count]
|
||||
if len(list) != len(new_list):
|
||||
msg += " List reduced from %r to %r due to restriction <%r>\n" % (
|
||||
len(list), len(new_list), sel)
|
||||
|
||||
return new_list, msg
|
||||
|
||||
def get_stats_profile(self):
|
||||
"""This method returns an instance of StatsProfile, which contains a mapping
|
||||
of function names to instances of FunctionProfile. Each FunctionProfile
|
||||
instance holds information related to the function's profile such as how
|
||||
long the function took to run, how many times it was called, etc...
|
||||
"""
|
||||
func_list = self.fcn_list[:] if self.fcn_list else list(self.stats.keys())
|
||||
if not func_list:
|
||||
return StatsProfile(0, {})
|
||||
|
||||
total_tt = float(f8(self.total_tt))
|
||||
func_profiles = {}
|
||||
stats_profile = StatsProfile(total_tt, func_profiles)
|
||||
|
||||
for func in func_list:
|
||||
cc, nc, tt, ct, callers = self.stats[func]
|
||||
file_name, line_number, func_name = func
|
||||
ncalls = str(nc) if nc == cc else (str(nc) + '/' + str(cc))
|
||||
tottime = float(f8(tt))
|
||||
percall_tottime = -1 if nc == 0 else float(f8(tt/nc))
|
||||
cumtime = float(f8(ct))
|
||||
percall_cumtime = -1 if cc == 0 else float(f8(ct/cc))
|
||||
func_profile = FunctionProfile(
|
||||
ncalls,
|
||||
tottime, # time spent in this function alone
|
||||
percall_tottime,
|
||||
cumtime, # time spent in the function plus all functions that this function called,
|
||||
percall_cumtime,
|
||||
file_name,
|
||||
line_number
|
||||
)
|
||||
func_profiles[func_name] = func_profile
|
||||
|
||||
return stats_profile
|
||||
|
||||
def get_print_list(self, sel_list):
|
||||
width = self.max_name_len
|
||||
if self.fcn_list:
|
||||
stat_list = self.fcn_list[:]
|
||||
msg = " Ordered by: " + self.sort_type + '\n'
|
||||
else:
|
||||
stat_list = list(self.stats.keys())
|
||||
msg = " Random listing order was used\n"
|
||||
|
||||
for selection in sel_list:
|
||||
stat_list, msg = self.eval_print_amount(selection, stat_list, msg)
|
||||
|
||||
count = len(stat_list)
|
||||
|
||||
if not stat_list:
|
||||
return 0, stat_list
|
||||
print(msg, file=self.stream)
|
||||
if count < len(self.stats):
|
||||
width = 0
|
||||
for func in stat_list:
|
||||
if len(func_std_string(func)) > width:
|
||||
width = len(func_std_string(func))
|
||||
return width+2, stat_list
|
||||
|
||||
def print_stats(self, *amount):
|
||||
for filename in self.files:
|
||||
print(filename, file=self.stream)
|
||||
if self.files:
|
||||
print(file=self.stream)
|
||||
indent = ' ' * 8
|
||||
for func in self.top_level:
|
||||
print(indent, func_get_function_name(func), file=self.stream)
|
||||
|
||||
print(indent, self.total_calls, "function calls", end=' ', file=self.stream)
|
||||
if self.total_calls != self.prim_calls:
|
||||
print("(%d primitive calls)" % self.prim_calls, end=' ', file=self.stream)
|
||||
print("in %.3f seconds" % self.total_tt, file=self.stream)
|
||||
print(file=self.stream)
|
||||
width, list = self.get_print_list(amount)
|
||||
if list:
|
||||
self.print_title()
|
||||
for func in list:
|
||||
self.print_line(func)
|
||||
print(file=self.stream)
|
||||
print(file=self.stream)
|
||||
return self
|
||||
|
||||
def print_callees(self, *amount):
|
||||
width, list = self.get_print_list(amount)
|
||||
if list:
|
||||
self.calc_callees()
|
||||
|
||||
self.print_call_heading(width, "called...")
|
||||
for func in list:
|
||||
if func in self.all_callees:
|
||||
self.print_call_line(width, func, self.all_callees[func])
|
||||
else:
|
||||
self.print_call_line(width, func, {})
|
||||
print(file=self.stream)
|
||||
print(file=self.stream)
|
||||
return self
|
||||
|
||||
def print_callers(self, *amount):
|
||||
width, list = self.get_print_list(amount)
|
||||
if list:
|
||||
self.print_call_heading(width, "was called by...")
|
||||
for func in list:
|
||||
cc, nc, tt, ct, callers = self.stats[func]
|
||||
self.print_call_line(width, func, callers, "<-")
|
||||
print(file=self.stream)
|
||||
print(file=self.stream)
|
||||
return self
|
||||
|
||||
def print_call_heading(self, name_size, column_title):
|
||||
print("Function ".ljust(name_size) + column_title, file=self.stream)
|
||||
# print sub-header only if we have new-style callers
|
||||
subheader = False
|
||||
for cc, nc, tt, ct, callers in self.stats.values():
|
||||
if callers:
|
||||
value = next(iter(callers.values()))
|
||||
subheader = isinstance(value, tuple)
|
||||
break
|
||||
if subheader:
|
||||
print(" "*name_size + " ncalls tottime cumtime", file=self.stream)
|
||||
|
||||
def print_call_line(self, name_size, source, call_dict, arrow="->"):
|
||||
print(func_std_string(source).ljust(name_size) + arrow, end=' ', file=self.stream)
|
||||
if not call_dict:
|
||||
print(file=self.stream)
|
||||
return
|
||||
clist = sorted(call_dict.keys())
|
||||
indent = ""
|
||||
for func in clist:
|
||||
name = func_std_string(func)
|
||||
value = call_dict[func]
|
||||
if isinstance(value, tuple):
|
||||
nc, cc, tt, ct = value
|
||||
if nc != cc:
|
||||
substats = '%d/%d' % (nc, cc)
|
||||
else:
|
||||
substats = '%d' % (nc,)
|
||||
substats = '%s %s %s %s' % (substats.rjust(7+2*len(indent)),
|
||||
f8(tt), f8(ct), name)
|
||||
left_width = name_size + 1
|
||||
else:
|
||||
substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3]))
|
||||
left_width = name_size + 3
|
||||
print(indent*left_width + substats, file=self.stream)
|
||||
indent = " "
|
||||
|
||||
def print_title(self):
|
||||
print(' ncalls tottime percall cumtime percall', end=' ', file=self.stream)
|
||||
print('filename:lineno(function)', file=self.stream)
|
||||
|
||||
def print_line(self, func): # hack: should print percentages
|
||||
cc, nc, tt, ct, callers = self.stats[func]
|
||||
c = str(nc)
|
||||
if nc != cc:
|
||||
c = c + '/' + str(cc)
|
||||
print(c.rjust(9), end=' ', file=self.stream)
|
||||
print(f8(tt), end=' ', file=self.stream)
|
||||
if nc == 0:
|
||||
print(' '*8, end=' ', file=self.stream)
|
||||
else:
|
||||
print(f8(tt/nc), end=' ', file=self.stream)
|
||||
print(f8(ct), end=' ', file=self.stream)
|
||||
if cc == 0:
|
||||
print(' '*8, end=' ', file=self.stream)
|
||||
else:
|
||||
print(f8(ct/cc), end=' ', file=self.stream)
|
||||
print(func_std_string(func), file=self.stream)
|
||||
|
||||
class TupleComp:
|
||||
"""This class provides a generic function for comparing any two tuples.
|
||||
Each instance records a list of tuple-indices (from most significant
|
||||
to least significant), and sort direction (ascending or descending) for
|
||||
each tuple-index. The compare functions can then be used as the function
|
||||
argument to the system sort() function when a list of tuples need to be
|
||||
sorted in the instances order."""
|
||||
|
||||
def __init__(self, comp_select_list):
|
||||
self.comp_select_list = comp_select_list
|
||||
|
||||
def compare (self, left, right):
|
||||
for index, direction in self.comp_select_list:
|
||||
l = left[index]
|
||||
r = right[index]
|
||||
if l < r:
|
||||
return -direction
|
||||
if l > r:
|
||||
return direction
|
||||
return 0
|
||||
|
||||
|
||||
#**************************************************************************
|
||||
# func_name is a triple (file:string, line:int, name:string)
|
||||
|
||||
def func_strip_path(func_name):
|
||||
filename, line, name = func_name
|
||||
return os.path.basename(filename), line, name
|
||||
|
||||
def func_get_function_name(func):
|
||||
return func[2]
|
||||
|
||||
def func_std_string(func_name): # match what old profile produced
|
||||
if func_name[:2] == ('~', 0):
|
||||
# special case for built-in functions
|
||||
name = func_name[2]
|
||||
if name.startswith('<') and name.endswith('>'):
|
||||
return '{%s}' % name[1:-1]
|
||||
else:
|
||||
return name
|
||||
else:
|
||||
return "%s:%d(%s)" % func_name
|
||||
|
||||
#**************************************************************************
|
||||
# The following functions combine statistics for pairs functions.
|
||||
# The bulk of the processing involves correctly handling "call" lists,
|
||||
# such as callers and callees.
|
||||
#**************************************************************************
|
||||
|
||||
def add_func_stats(target, source):
|
||||
"""Add together all the stats for two profile entries."""
|
||||
cc, nc, tt, ct, callers = source
|
||||
t_cc, t_nc, t_tt, t_ct, t_callers = target
|
||||
return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,
|
||||
add_callers(t_callers, callers))
|
||||
|
||||
def add_callers(target, source):
|
||||
"""Combine two caller lists in a single list."""
|
||||
new_callers = {}
|
||||
for func, caller in target.items():
|
||||
new_callers[func] = caller
|
||||
for func, caller in source.items():
|
||||
if func in new_callers:
|
||||
if isinstance(caller, tuple):
|
||||
# format used by cProfile
|
||||
new_callers[func] = tuple(i + j for i, j in zip(caller, new_callers[func]))
|
||||
else:
|
||||
# format used by profile
|
||||
new_callers[func] += caller
|
||||
else:
|
||||
new_callers[func] = caller
|
||||
return new_callers
|
||||
|
||||
def count_calls(callers):
|
||||
"""Sum the caller statistics to get total number of calls received."""
|
||||
nc = 0
|
||||
for calls in callers.values():
|
||||
nc += calls
|
||||
return nc
|
||||
|
||||
#**************************************************************************
|
||||
# The following functions support printing of reports
|
||||
#**************************************************************************
|
||||
|
||||
def f8(x):
|
||||
return "%8.3f" % x
|
||||
|
||||
#**************************************************************************
|
||||
# Statistics browser added by ESR, April 2001
|
||||
#**************************************************************************
|
||||
|
||||
if __name__ == '__main__':
|
||||
import cmd
|
||||
try:
|
||||
import readline # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
class ProfileBrowser(cmd.Cmd):
|
||||
def __init__(self, profile=None):
|
||||
cmd.Cmd.__init__(self)
|
||||
self.prompt = "% "
|
||||
self.stats = None
|
||||
self.stream = sys.stdout
|
||||
if profile is not None:
|
||||
self.do_read(profile)
|
||||
|
||||
def generic(self, fn, line):
|
||||
args = line.split()
|
||||
processed = []
|
||||
for term in args:
|
||||
try:
|
||||
processed.append(int(term))
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
frac = float(term)
|
||||
if frac > 1 or frac < 0:
|
||||
print("Fraction argument must be in [0, 1]", file=self.stream)
|
||||
continue
|
||||
processed.append(frac)
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
processed.append(term)
|
||||
if self.stats:
|
||||
getattr(self.stats, fn)(*processed)
|
||||
else:
|
||||
print("No statistics object is loaded.", file=self.stream)
|
||||
return 0
|
||||
def generic_help(self):
|
||||
print("Arguments may be:", file=self.stream)
|
||||
print("* An integer maximum number of entries to print.", file=self.stream)
|
||||
print("* A decimal fractional number between 0 and 1, controlling", file=self.stream)
|
||||
print(" what fraction of selected entries to print.", file=self.stream)
|
||||
print("* A regular expression; only entries with function names", file=self.stream)
|
||||
print(" that match it are printed.", file=self.stream)
|
||||
|
||||
def do_add(self, line):
|
||||
if self.stats:
|
||||
try:
|
||||
self.stats.add(line)
|
||||
except OSError as e:
|
||||
print("Failed to load statistics for %s: %s" % (line, e), file=self.stream)
|
||||
else:
|
||||
print("No statistics object is loaded.", file=self.stream)
|
||||
return 0
|
||||
def help_add(self):
|
||||
print("Add profile info from given file to current statistics object.", file=self.stream)
|
||||
|
||||
def do_callees(self, line):
|
||||
return self.generic('print_callees', line)
|
||||
def help_callees(self):
|
||||
print("Print callees statistics from the current stat object.", file=self.stream)
|
||||
self.generic_help()
|
||||
|
||||
def do_callers(self, line):
|
||||
return self.generic('print_callers', line)
|
||||
def help_callers(self):
|
||||
print("Print callers statistics from the current stat object.", file=self.stream)
|
||||
self.generic_help()
|
||||
|
||||
def do_EOF(self, line):
|
||||
print("", file=self.stream)
|
||||
return 1
|
||||
def help_EOF(self):
|
||||
print("Leave the profile browser.", file=self.stream)
|
||||
|
||||
def do_quit(self, line):
|
||||
return 1
|
||||
def help_quit(self):
|
||||
print("Leave the profile browser.", file=self.stream)
|
||||
|
||||
def do_read(self, line):
|
||||
if line:
|
||||
try:
|
||||
self.stats = Stats(line)
|
||||
except OSError as err:
|
||||
print(err.args[1], file=self.stream)
|
||||
return
|
||||
except Exception as err:
|
||||
print(err.__class__.__name__ + ':', err, file=self.stream)
|
||||
return
|
||||
self.prompt = line + "% "
|
||||
elif len(self.prompt) > 2:
|
||||
line = self.prompt[:-2]
|
||||
self.do_read(line)
|
||||
else:
|
||||
print("No statistics object is current -- cannot reload.", file=self.stream)
|
||||
return 0
|
||||
def help_read(self):
|
||||
print("Read in profile data from a specified file.", file=self.stream)
|
||||
print("Without argument, reload the current file.", file=self.stream)
|
||||
|
||||
def do_reverse(self, line):
|
||||
if self.stats:
|
||||
self.stats.reverse_order()
|
||||
else:
|
||||
print("No statistics object is loaded.", file=self.stream)
|
||||
return 0
|
||||
def help_reverse(self):
|
||||
print("Reverse the sort order of the profiling report.", file=self.stream)
|
||||
|
||||
def do_sort(self, line):
|
||||
if not self.stats:
|
||||
print("No statistics object is loaded.", file=self.stream)
|
||||
return
|
||||
abbrevs = self.stats.get_sort_arg_defs()
|
||||
if line and all((x in abbrevs) for x in line.split()):
|
||||
self.stats.sort_stats(*line.split())
|
||||
else:
|
||||
print("Valid sort keys (unique prefixes are accepted):", file=self.stream)
|
||||
for (key, value) in Stats.sort_arg_dict_default.items():
|
||||
print("%s -- %s" % (key, value[1]), file=self.stream)
|
||||
return 0
|
||||
def help_sort(self):
|
||||
print("Sort profile data according to specified keys.", file=self.stream)
|
||||
print("(Typing `sort' without arguments lists valid keys.)", file=self.stream)
|
||||
def complete_sort(self, text, *args):
|
||||
return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]
|
||||
|
||||
def do_stats(self, line):
|
||||
return self.generic('print_stats', line)
|
||||
def help_stats(self):
|
||||
print("Print statistics from the current stat object.", file=self.stream)
|
||||
self.generic_help()
|
||||
|
||||
def do_strip(self, line):
|
||||
if self.stats:
|
||||
self.stats.strip_dirs()
|
||||
else:
|
||||
print("No statistics object is loaded.", file=self.stream)
|
||||
def help_strip(self):
|
||||
print("Strip leading path information from filenames in the report.", file=self.stream)
|
||||
|
||||
def help_help(self):
|
||||
print("Show help for a given command.", file=self.stream)
|
||||
|
||||
def postcmd(self, stop, line):
|
||||
if stop:
|
||||
return stop
|
||||
return None
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
initprofile = sys.argv[1]
|
||||
else:
|
||||
initprofile = None
|
||||
try:
|
||||
browser = ProfileBrowser(initprofile)
|
||||
for profile in sys.argv[2:]:
|
||||
browser.do_add(profile)
|
||||
print("Welcome to the profile statistics browser.", file=browser.stream)
|
||||
browser.cmdloop()
|
||||
print("Goodbye.", file=browser.stream)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
# That's all, folks.
|
||||
47
Lib/pty.py
vendored
47
Lib/pty.py
vendored
@@ -32,27 +32,18 @@ def openpty():
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
master_fd, slave_name = _open_terminal()
|
||||
slave_fd = slave_open(slave_name)
|
||||
return master_fd, slave_fd
|
||||
|
||||
def master_open():
|
||||
"""master_open() -> (master_fd, slave_name)
|
||||
Open a pty master and return the fd, and the filename of the slave end.
|
||||
Deprecated, use openpty() instead."""
|
||||
|
||||
import warnings
|
||||
warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14
|
||||
|
||||
slave_fd = os.open(slave_name, os.O_RDWR)
|
||||
try:
|
||||
master_fd, slave_fd = os.openpty()
|
||||
except (AttributeError, OSError):
|
||||
from fcntl import ioctl, I_PUSH
|
||||
except ImportError:
|
||||
return master_fd, slave_fd
|
||||
try:
|
||||
ioctl(slave_fd, I_PUSH, "ptem")
|
||||
ioctl(slave_fd, I_PUSH, "ldterm")
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
slave_name = os.ttyname(slave_fd)
|
||||
os.close(slave_fd)
|
||||
return master_fd, slave_name
|
||||
|
||||
return _open_terminal()
|
||||
return master_fd, slave_fd
|
||||
|
||||
def _open_terminal():
|
||||
"""Open pty master and return (master_fd, tty_name)."""
|
||||
@@ -66,26 +57,6 @@ def _open_terminal():
|
||||
return (fd, '/dev/tty' + x + y)
|
||||
raise OSError('out of pty devices')
|
||||
|
||||
def slave_open(tty_name):
|
||||
"""slave_open(tty_name) -> slave_fd
|
||||
Open the pty slave and acquire the controlling terminal, returning
|
||||
opened filedescriptor.
|
||||
Deprecated, use openpty() instead."""
|
||||
|
||||
import warnings
|
||||
warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14
|
||||
|
||||
result = os.open(tty_name, os.O_RDWR)
|
||||
try:
|
||||
from fcntl import ioctl, I_PUSH
|
||||
except ImportError:
|
||||
return result
|
||||
try:
|
||||
ioctl(result, I_PUSH, "ptem")
|
||||
ioctl(result, I_PUSH, "ldterm")
|
||||
except OSError:
|
||||
pass
|
||||
return result
|
||||
|
||||
def fork():
|
||||
"""fork() -> (pid, master_fd)
|
||||
|
||||
2
Lib/pydoc_data/module_docs.py
vendored
2
Lib/pydoc_data/module_docs.py
vendored
@@ -1,4 +1,4 @@
|
||||
# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026
|
||||
# Autogenerated by Sphinx on Sun May 10 13:21:26 2026
|
||||
# as part of the release process.
|
||||
|
||||
module_docs = {
|
||||
|
||||
909
Lib/pydoc_data/topics.py
vendored
909
Lib/pydoc_data/topics.py
vendored
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user