src/pyparsing/common.py
1# common.py
2from .core import *
3from .helpers import DelimitedList, any_open_tag, any_close_tag
4from datetime import datetime
5import sys
6
7PY_310_OR_LATER = sys.version_info >= (3, 10)
8
9
10# some other useful expressions - using lower-case class name since we are really using this as a namespace
11class pyparsing_common:
12 """Here are some common low-level expressions that may be useful in
13 jump-starting parser development:
14
15 - numeric forms (:class:`integers<integer>`, :class:`reals<real>`,
16 :class:`scientific notation<sci_real>`)
17 - common :class:`programming identifiers<identifier>`
18 - network addresses (:class:`MAC<mac_address>`,
19 :class:`IPv4<ipv4_address>`, :class:`IPv6<ipv6_address>`)
20 - ISO8601 :class:`dates<iso8601_date>` and
21 :class:`datetime<iso8601_datetime>`
22 - :class:`UUID<uuid>`
23 - :class:`comma-separated list<comma_separated_list>`
24 - :class:`url`
25
26 Parse actions:
27
28 - :class:`convert_to_integer`
29 - :class:`convert_to_float`
30 - :class:`convert_to_date`
31 - :class:`convert_to_datetime`
32 - :class:`strip_html_tags`
33 - :class:`upcase_tokens`
34 - :class:`downcase_tokens`
35
36 Examples:
37
38 .. testcode::
39
40 pyparsing_common.number.run_tests('''
41 # any int or real number, returned as the appropriate type
42 100
43 -100
44 +100
45 3.14159
46 6.02e23
47 1e-12
48 ''')
49
50 .. testoutput::
51 :options: +NORMALIZE_WHITESPACE
52
53
54 # any int or real number, returned as the appropriate type
55 100
56 [100]
57
58 -100
59 [-100]
60
61 +100
62 [100]
63
64 3.14159
65 [3.14159]
66
67 6.02e23
68 [6.02e+23]
69
70 1e-12
71 [1e-12]
72
73 .. testcode::
74
75 pyparsing_common.fnumber.run_tests('''
76 # any int or real number, returned as float
77 100
78 -100
79 +100
80 3.14159
81 6.02e23
82 1e-12
83 ''')
84
85 .. testoutput::
86 :options: +NORMALIZE_WHITESPACE
87
88
89 # any int or real number, returned as float
90 100
91 [100.0]
92
93 -100
94 [-100.0]
95
96 +100
97 [100.0]
98
99 3.14159
100 [3.14159]
101
102 6.02e23
103 [6.02e+23]
104
105 1e-12
106 [1e-12]
107
108 .. testcode::
109
110 pyparsing_common.hex_integer.run_tests('''
111 # hex numbers
112 100
113 FF
114 ''')
115
116 .. testoutput::
117 :options: +NORMALIZE_WHITESPACE
118
119
120 # hex numbers
121 100
122 [256]
123
124 FF
125 [255]
126
127 .. testcode::
128
129 pyparsing_common.fraction.run_tests('''
130 # fractions
131 1/2
132 -3/4
133 ''')
134
135 .. testoutput::
136 :options: +NORMALIZE_WHITESPACE
137
138
139 # fractions
140 1/2
141 [0.5]
142
143 -3/4
144 [-0.75]
145
146 .. testcode::
147
148 pyparsing_common.mixed_integer.run_tests('''
149 # mixed fractions
150 1
151 1/2
152 -3/4
153 1-3/4
154 ''')
155
156 .. testoutput::
157 :options: +NORMALIZE_WHITESPACE
158
159
160 # mixed fractions
161 1
162 [1]
163
164 1/2
165 [0.5]
166
167 -3/4
168 [-0.75]
169
170 1-3/4
171 [1.75]
172 .. testcode::
173
174 import uuid
175 pyparsing_common.uuid.set_parse_action(token_map(uuid.UUID))
176 pyparsing_common.uuid.run_tests('''
177 # uuid
178 12345678-1234-5678-1234-567812345678
179 ''')
180
181 .. testoutput::
182 :options: +NORMALIZE_WHITESPACE
183
184
185 # uuid
186 12345678-1234-5678-1234-567812345678
187 [UUID('12345678-1234-5678-1234-567812345678')]
188 """
189
190 @staticmethod
191 def convert_to_integer(_, __, t):
192 """
193 Parse action for converting parsed integers to Python int
194 """
195 return [int(tt) for tt in t]
196
197 @staticmethod
198 def convert_to_float(_, __, t):
199 """
200 Parse action for converting parsed numbers to Python float
201 """
202 return [float(tt) for tt in t]
203
204 integer = (
205 Word(nums)
206 .set_name("integer")
207 .set_parse_action(
208 convert_to_integer
209 if PY_310_OR_LATER
210 else lambda t: [int(tt) for tt in t] # type: ignore[misc]
211 )
212 )
213 """expression that parses an unsigned integer, converts to an int"""
214
215 hex_integer = (
216 Word(hexnums).set_name("hex integer").set_parse_action(token_map(int, 16))
217 )
218 """expression that parses a hexadecimal integer, converts to an int"""
219
220 signed_integer = (
221 Regex(r"[+-]?\d+")
222 .set_name("signed integer")
223 .set_parse_action(
224 convert_to_integer
225 if PY_310_OR_LATER
226 else lambda t: [int(tt) for tt in t] # type: ignore[misc]
227 )
228 )
229 """expression that parses an integer with optional leading sign, converts to an int"""
230
231 fraction = (
232 signed_integer().set_parse_action(
233 convert_to_float
234 if PY_310_OR_LATER
235 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
236 )
237 + "/"
238 + signed_integer().set_parse_action(
239 convert_to_float
240 if PY_310_OR_LATER
241 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
242 )
243 ).set_name("fraction")
244 """fractional expression of an integer divided by an integer, converts to a float"""
245 fraction.add_parse_action(lambda tt: tt[0] / tt[-1])
246
247 mixed_integer = (
248 fraction | signed_integer + Opt(Opt("-").suppress() + fraction)
249 ).set_name("fraction or mixed integer-fraction")
250 """mixed integer of the form 'integer - fraction', with optional leading integer, converts to a float"""
251 mixed_integer.add_parse_action(sum)
252
253 real = (
254 Regex(r"[+-]?(?:\d+\.\d*|\.\d+)")
255 .set_name("real number")
256 .set_parse_action(
257 convert_to_float
258 if PY_310_OR_LATER
259 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
260 )
261 )
262 """expression that parses a floating point number, converts to a float"""
263
264 sci_real = (
265 Regex(r"[+-]?(?:\d+(?:[eE][+-]?\d+)|(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?)")
266 .set_name("real number with scientific notation")
267 .set_parse_action(
268 convert_to_float
269 if PY_310_OR_LATER
270 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
271 )
272 )
273 """expression that parses a floating point number with optional
274 scientific notation, converts to a float"""
275
276 # streamlining this expression makes the docs nicer-looking
277 number = (sci_real | real | signed_integer).set_name("number").streamline()
278 """any numeric expression, converts to the corresponding Python type"""
279
280 fnumber = (
281 Regex(r"[+-]?\d+\.?\d*(?:[eE][+-]?\d+)?")
282 .set_name("fnumber")
283 .set_parse_action(
284 convert_to_float
285 if PY_310_OR_LATER
286 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
287 )
288 )
289 """any int or real number, always converts to a float"""
290
291 ieee_float = (
292 Regex(r"(?i:[+-]?(?:(?:\d+\.?\d*(?:e[+-]?\d+)?)|nan|inf(?:inity)?))")
293 .set_name("ieee_float")
294 .set_parse_action(
295 convert_to_float
296 if PY_310_OR_LATER
297 else lambda t: [float(tt) for tt in t] # type: ignore[misc]
298 )
299 )
300 """any floating-point literal (int, real number, infinity, or NaN), converts to a float"""
301
302 identifier = Word(identchars, identbodychars).set_name("identifier")
303 """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
304
305 ipv4_address = Regex(
306 r"(?:25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(?:\.(?:25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}"
307 ).set_name("IPv4 address")
308 "IPv4 address (``0.0.0.0 - 255.255.255.255``)"
309
310 _ipv6_part = Regex(r"[0-9a-fA-F]{1,4}").set_name("hex_integer")
311 _full_ipv6_address = (_ipv6_part + (":" + _ipv6_part) * 7).set_name(
312 "full IPv6 address"
313 )
314 _short_ipv6_address = (
315 Opt(_ipv6_part + (":" + _ipv6_part) * (0, 6))
316 + "::"
317 + Opt(_ipv6_part + (":" + _ipv6_part) * (0, 6))
318 ).set_name("short IPv6 address")
319 _short_ipv6_address.add_condition(
320 lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8
321 )
322 _mixed_ipv6_address = ("::ffff:" + ipv4_address).set_name("mixed IPv6 address")
323 ipv6_address = Combine(
324 (_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).set_name(
325 "IPv6 address"
326 )
327 ).set_name("IPv6 address")
328 "IPv6 address (long, short, or mixed form)"
329
330 mac_address = Regex(
331 r"[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}"
332 ).set_name("MAC address")
333 "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
334
335 @staticmethod
336 def convert_to_date(fmt: str = "%Y-%m-%d"):
337 """
338 Helper to create a parse action for converting parsed date string to Python datetime.date
339
340 Params -
341 - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``)
342
343 Example:
344
345 .. testcode::
346
347 date_expr = pyparsing_common.iso8601_date.copy()
348 date_expr.set_parse_action(pyparsing_common.convert_to_date())
349 print(date_expr.parse_string("1999-12-31"))
350
351 prints:
352
353 .. testoutput::
354
355 [datetime.date(1999, 12, 31)]
356 """
357
358 def cvt_fn(ss, ll, tt):
359 try:
360 return datetime.strptime(tt[0], fmt).date()
361 except ValueError as ve:
362 raise ParseException(ss, ll, str(ve))
363
364 return cvt_fn
365
366 @staticmethod
367 def convert_to_datetime(fmt: str = "%Y-%m-%dT%H:%M:%S.%f"):
368 """Helper to create a parse action for converting parsed
369 datetime string to Python :class:`datetime.datetime`
370
371 Params -
372 - fmt - format to be passed to :class:`datetime.strptime` (default= ``"%Y-%m-%dT%H:%M:%S.%f"``)
373
374 Example:
375
376 .. testcode::
377
378 dt_expr = pyparsing_common.iso8601_datetime.copy()
379 dt_expr.set_parse_action(pyparsing_common.convert_to_datetime())
380 print(dt_expr.parse_string("1999-12-31T23:59:59.999"))
381
382 prints:
383
384 .. testoutput::
385
386 [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
387 """
388
389 def cvt_fn(s, l, t):
390 try:
391 return datetime.strptime(t[0], fmt)
392 except ValueError as ve:
393 raise ParseException(s, l, str(ve))
394
395 return cvt_fn
396
397 iso8601_date = Regex(
398 r"(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?"
399 ).set_name("ISO8601 date")
400 "ISO8601 date (``yyyy-mm-dd``)"
401
402 iso8601_datetime = Regex(
403 r"(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?"
404 ).set_name("ISO8601 datetime")
405 "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``"
406
407 @staticmethod
408 def as_datetime(s, l, t):
409 """Parse action to convert parsed dates or datetimes to a Python
410 :class:`datetime.datetime`.
411
412 This parse action will use the year, month, day, etc. results
413 names defined in the ISO8601 date expressions, but it can be
414 used with any expression that provides one or more of these fields.
415
416 Omitted fields will default to fields from Jan 1, 00:00:00.
417
418 Invalid dates will raise a :class:`ParseException` with the
419 error message indicating the invalid date fields.
420 """
421 year = int(t.year.lstrip("0") or 0)
422 month = int(t.month or 1)
423 day = int(t.day or 1)
424 hour = int(t.hour or 0)
425 minute = int(t.minute or 0)
426 second = float(t.second or 0)
427 try:
428 return datetime(
429 year, month, day, hour, minute, int(second), int((second % 1) * 1000)
430 )
431 except ValueError as ve:
432 raise ParseException(t, l, f"Invalid date/time: {ve}").with_traceback(
433 ve.__traceback__
434 ) from None
435
436 if PY_310_OR_LATER:
437 iso8601_date_validated = iso8601_date().add_parse_action(as_datetime)
438 "Validated ISO8601 date strings, raising :class:`ParseException` for invalid date values."
439
440 iso8601_datetime_validated = iso8601_datetime().add_parse_action(as_datetime)
441 "Validated ISO8601 date and time strings, raising :class:`ParseException` for invalid date/time values."
442
443 uuid = Regex(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}").set_name(
444 "UUID"
445 )
446 "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)"
447
448 _html_stripper = any_open_tag.suppress() | any_close_tag.suppress()
449
450 @staticmethod
451 def strip_html_tags(s: str, l: int, tokens: ParseResults):
452 """Parse action to remove HTML tags from web page HTML source
453
454 Example:
455
456 .. testcode::
457
458 # strip HTML links from normal text
459 text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>'
460 td, td_end = make_html_tags("TD")
461 table_text = td + SkipTo(td_end).set_parse_action(
462 pyparsing_common.strip_html_tags)("body") + td_end
463 print(table_text.parse_string(text).body)
464
465 Prints:
466
467 .. testoutput::
468
469 More info at the pyparsing wiki page
470 """
471 return pyparsing_common._html_stripper.transform_string(tokens[0])
472
473 _commasepitem = (
474 Combine(
475 OneOrMore(
476 ~Literal(",")
477 + ~LineEnd()
478 + Word(printables, exclude_chars=",")
479 + Opt(White(" \t") + ~FollowedBy(LineEnd() | ","))
480 )
481 )
482 .streamline()
483 .set_name("commaItem")
484 )
485 comma_separated_list = DelimitedList(
486 Opt(quoted_string.copy() | _commasepitem, default="")
487 ).set_name("comma separated list")
488 """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
489
490 @staticmethod
491 def upcase_tokens(s, l, t):
492 """Parse action to convert tokens to upper case."""
493 return [tt.upper() for tt in t]
494
495 @staticmethod
496 def downcase_tokens(s, l, t):
497 """Parse action to convert tokens to lower case."""
498 return [tt.lower() for tt in t]
499
500 # fmt: off
501 url = Regex(
502 # https://mathiasbynens.be/demo/url-regex
503 # https://gist.github.com/dperini/729294
504 r"(?P<url>"
505 # protocol identifier (optional)
506 # short syntax // still required
507 r"(?:(?:(?P<scheme>https?|ftp):)?\/\/)"
508 # user:pass BasicAuth (optional)
509 r"(?:(?P<auth>\S+(?::\S*)?)@)?"
510 r"(?P<host>"
511 # IP address exclusion
512 # private & local networks
513 r"(?!(?:10|127)(?:\.\d{1,3}){3})"
514 r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
515 r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
516 # IP address dotted notation octets
517 # excludes loopback network 0.0.0.0
518 # excludes reserved space >= 224.0.0.0
519 # excludes network & broadcast addresses
520 # (first & last IP address of each class)
521 r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
522 r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
523 r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
524 r"|"
525 # host & domain names, may end with dot
526 # can be replaced by a shortest alternative
527 # (?![-_])(?:[-\w\u00a1-\uffff]{0,63}[^-_]\.)+
528 r"(?:"
529 r"(?:"
530 r"[a-z0-9\u00a1-\uffff]"
531 r"[a-z0-9\u00a1-\uffff_-]{0,62}"
532 r")?"
533 r"[a-z0-9\u00a1-\uffff]\."
534 r")+"
535 # TLD identifier name, may end with dot
536 r"(?:[a-z\u00a1-\uffff]{2,}\.?)"
537 r")"
538 # port number (optional)
539 r"(:(?P<port>\d{2,5}))?"
540 # resource path (optional)
541 r"(?P<path>\/[^?# ]*)?"
542 # query string (optional)
543 r"(\?(?P<query>[^#]*))?"
544 # fragment (optional)
545 r"(#(?P<fragment>\S*))?"
546 r")"
547 ).set_name("url")
548 """
549 URL (http/https/ftp scheme)
550
551 .. versionchanged:: 3.1.0
552 ``url`` named group added
553 """
554 # fmt: on
555
556 # pre-PEP8 compatibility names
557 # fmt: off
558 convertToInteger = staticmethod(replaced_by_pep8("convertToInteger", convert_to_integer))
559 convertToFloat = staticmethod(replaced_by_pep8("convertToFloat", convert_to_float))
560 convertToDate = staticmethod(replaced_by_pep8("convertToDate", convert_to_date))
561 convertToDatetime = staticmethod(replaced_by_pep8("convertToDatetime", convert_to_datetime))
562 stripHTMLTags = staticmethod(replaced_by_pep8("stripHTMLTags", strip_html_tags))
563 upcaseTokens = staticmethod(replaced_by_pep8("upcaseTokens", upcase_tokens))
564 downcaseTokens = staticmethod(replaced_by_pep8("downcaseTokens", downcase_tokens))
565 # fmt: on
566
567
568_builtin_exprs = [
569 v for v in vars(pyparsing_common).values() if isinstance(v, ParserElement)
570]