Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:jayvdb:django
python-pyxb
pr_97.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File pr_97.patch of Package python-pyxb
From b25f37b601f101e1cd34a3c7131fa265dad61ba2 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Fri, 23 Feb 2018 15:01:21 +0000 Subject: [PATCH 01/10] Improve XML Regex code Rework XML regex code to follow the standards. It now completely parses the XML regex and generates a corresponding Python regex, instead of relying on the syntaxes being "close enough". --- pyxb/utils/unicode.py | 89 +++++++++---- pyxb/utils/xmlre.py | 274 +++++++++++++++++++++++++++++++------- tests/utils/test-xmlre.py | 79 ++++++----- 3 files changed, 323 insertions(+), 119 deletions(-) diff --git a/pyxb/utils/unicode.py b/pyxb/utils/unicode.py index e330c982..e2dacc2c 100644 --- a/pyxb/utils/unicode.py +++ b/pyxb/utils/unicode.py @@ -80,8 +80,11 @@ class is used to represent a set of code points in a manner # codepoints 12, 13, 14, and everything above 199. __codepoints = None + # Whether this object is frozen (immutable) + __frozen = False + def _codepoints (self): - """For testing purrposes only, access to the codepoints + """For testing purposes only, access to the codepoints internal representation.""" return self.__codepoints @@ -90,7 +93,7 @@ def __hash__ (self): def __eq__ (self, other): """Equality is delegated to the codepoints list.""" - return self.__codepoints == other.__codepoints + return isinstance(other, CodePointSet) and self.__codepoints == other.__codepoints def __lt__ (self, other): return self.__codepoints < other.__codepoints @@ -113,7 +116,7 @@ def __mutate (self, value, do_add): (s, e) = value e += 1 elif isinstance(value, six.string_types): - if 1 < len(value): + if 1 != len(value): raise TypeError() s = ord(value) e = s+1 @@ -125,10 +128,12 @@ def __mutate (self, value, do_add): # Validate the range for the code points supported by this # Python interpreter. Recall that e is exclusive. - if s > self.MaxCodePoint: + if s > self.MaxCodePoint or e <= 0: return self + if s < 0: + s = 0 if e > self.MaxCodePoint: - e = self.MaxCodePoint+1 + e = self.MaxCodePoint + 1 # Index of first code point equal to or greater than s li = bisect.bisect_left(self.__codepoints, s) @@ -161,6 +166,7 @@ def add (self, value): tuple C{(s,e)} denoting the start and end (inclusive) code points in a range. @return: C{self}""" + assert not self.__frozen return self.__mutate(value, True) def extend (self, values): @@ -170,6 +176,7 @@ def extend (self, values): whose members are valid parameters to L{add}. @return: C{self}""" + assert not self.__frozen if isinstance(values, CodePointSet): self.extend(values.asTuples()) else: @@ -185,6 +192,7 @@ def subtract (self, value): range, or a L{CodePointSet}. @return: C{self}""" + assert not self.__frozen if isinstance(value, CodePointSet): for v in value.asTuples(): self.subtract(v) @@ -217,7 +225,7 @@ def subtract (self, value): def __unichr (self, code_point): rv = six.unichr(code_point) if 0 == code_point: - rv = six.u('x00') + rv = six.text_type('x00') if code_point in self.__XMLtoPythonREEscapedCodepoints: rv = six.unichr(0x5c) + rv return rv @@ -240,7 +248,7 @@ def asPattern (self, with_brackets=True): if s == e: rva.append(self.__unichr(s)) else: - rva.extend([self.__unichr(s), '-', self.__unichr(e)]) + rva.extend([self.__unichr(s), six.u('-'), self.__unichr(e)]) if with_brackets: rva.append(six.u(']')) return six.u('').join(rva) @@ -281,6 +289,22 @@ def asSingleCharacter (self): return None return six.unichr(self.__codepoints[0]) + def freeze (self): + """Freezes this object, preventing further changes. + + Attempting to modify a frozen object will assert if assertions are + enabled. If assertions are disabled then you can still modify the + object. + + Intended as a development/debugging aid, to prevent accidentally + changing objects that are used as global constants. + + @return: C{self} + """ + self.__frozen = True + return self + + from pyxb.utils.unicode_data import PropertyMap from pyxb.utils.unicode_data import BlockMap @@ -316,6 +340,7 @@ class XML1p0e2 (object): ) if SupportsWideUnicode: Char.add( ( 1+CodePointSet.MaxShortCodePoint, CodePointSet.MaxCodePoint ) ) + Char.freeze() BaseChar = CodePointSet( ( 0x0041, 0x005A ), @@ -520,15 +545,15 @@ class XML1p0e2 (object): ( 0x30A1, 0x30FA ), ( 0x3105, 0x312C ), ( 0xAC00, 0xD7A3 ) - ) + ).freeze() Ideographic = CodePointSet( ( 0x4E00, 0x9FA5 ), 0x3007, ( 0x3021, 0x3029 ) - ) + ).freeze() - Letter = CodePointSet(BaseChar).extend(Ideographic) + Letter = CodePointSet(BaseChar).extend(Ideographic).freeze() CombiningChar = CodePointSet( ( 0x0300, 0x0345 ), @@ -626,7 +651,7 @@ class XML1p0e2 (object): ( 0x302A, 0x302F ), 0x3099, 0x309A - ) + ).freeze() Digit = CodePointSet( ( 0x0030, 0x0039 ), @@ -644,7 +669,7 @@ class XML1p0e2 (object): ( 0x0E50, 0x0E59 ), ( 0x0ED0, 0x0ED9 ), ( 0x0F20, 0x0F29 ) - ) + ).freeze() Extender = CodePointSet( 0x00B7, @@ -658,15 +683,17 @@ class XML1p0e2 (object): ( 0x3031, 0x3035 ), ( 0x309D, 0x309E ), ( 0x30FC, 0x30FE ) - ) + ).freeze() # Not an explicit production, but used in Name production NameStartChar = CodePointSet(Letter) NameStartChar.add(ord('_')) NameStartChar.add(ord(':')) + NameStartChar.freeze() NCNameStartChar = CodePointSet(Letter) NCNameStartChar.add(ord('_')) + NCNameStartChar.freeze() NameChar = CodePointSet(Letter) NameChar.extend(Digit) @@ -676,6 +703,7 @@ class XML1p0e2 (object): NameChar.add(ord(':')) NameChar.extend(CombiningChar) NameChar.extend(Extender) + NameChar.freeze() NCNameChar = CodePointSet(Letter) NCNameChar.extend(Digit) @@ -684,6 +712,7 @@ class XML1p0e2 (object): NCNameChar.add(ord('_')) NCNameChar.extend(CombiningChar) NCNameChar.extend(Extender) + NCNameChar.freeze() Name_pat = '%s%s*' % (NameStartChar.asPattern(), NameChar.asPattern()) Name_re = re.compile('^%s$' % (Name_pat,)) @@ -695,36 +724,38 @@ class XML1p0e2 (object): QName_re = re.compile('^%s$' % (QName_pat,)) # Production 24 : Single Character Escapes -SingleCharEsc = { 'n' : CodePointSet(0x0A), - 'r' : CodePointSet(0x0D), - 't' : CodePointSet(0x09) } +SingleCharEsc = { 'n' : CodePointSet(0x0A).freeze(), + 'r' : CodePointSet(0x0D).freeze(), + 't' : CodePointSet(0x09).freeze() } for c in r'\|.-^?*+{}()[]': - SingleCharEsc[c] = CodePointSet(ord(c)) + SingleCharEsc[c] = CodePointSet(ord(c)).freeze() # Production 25 : Category Escapes # Production 26: Complemented Category Escapes catEsc = { } complEsc = { } for k, v in six.iteritems(PropertyMap): + v.freeze() catEsc[six.u('p{%s}') % (k,)] = v - catEsc[six.u('P{%s}') % (k,)] = v.negate() + complEsc[six.u('P{%s}') % (k,)] = v.negate().freeze() # Production 36 : IsBlock escapes IsBlockEsc = { } for k, v in six.iteritems(BlockMap): + v.freeze() IsBlockEsc[six.u('p{Is%s}') % (k,)] = v - IsBlockEsc[six.u('P{Is%s}') % (k,)] = v.negate() + IsBlockEsc[six.u('P{Is%s}') % (k,)] = v.negate().freeze() # Production 37 : Multi-Character Escapes -WildcardEsc = CodePointSet(ord('\n'), ord('\r')).negate() +WildcardEsc = CodePointSet(ord('\n'), ord('\r')).negate().freeze() MultiCharEsc = { } -MultiCharEsc['s'] = CodePointSet(0x20, ord('\t'), ord('\n'), ord('\r')) -MultiCharEsc['S'] = MultiCharEsc['s'].negate() -MultiCharEsc['i'] = CodePointSet(XML1p0e2.Letter).add(ord('_')).add(ord(':')) -MultiCharEsc['I'] = MultiCharEsc['i'].negate() -MultiCharEsc['c'] = CodePointSet(XML1p0e2.NameChar) -MultiCharEsc['C'] = MultiCharEsc['c'].negate() +MultiCharEsc['s'] = CodePointSet(0x20, ord('\t'), ord('\n'), ord('\r')).freeze() +MultiCharEsc['S'] = MultiCharEsc['s'].negate().freeze() +MultiCharEsc['i'] = CodePointSet(XML1p0e2.Letter).add(ord('_')).add(ord(':')).freeze() +MultiCharEsc['I'] = MultiCharEsc['i'].negate().freeze() +MultiCharEsc['c'] = CodePointSet(XML1p0e2.NameChar).freeze() +MultiCharEsc['C'] = MultiCharEsc['c'].negate().freeze() MultiCharEsc['d'] = PropertyMap['Nd'] -MultiCharEsc['D'] = MultiCharEsc['d'].negate() -MultiCharEsc['W'] = CodePointSet(PropertyMap['P']).extend(PropertyMap['Z']).extend(PropertyMap['C']) -MultiCharEsc['w'] = MultiCharEsc['W'].negate() +MultiCharEsc['D'] = MultiCharEsc['d'].negate().freeze() +MultiCharEsc['W'] = CodePointSet(PropertyMap['P']).extend(PropertyMap['Z']).extend(PropertyMap['C']).freeze() +MultiCharEsc['w'] = MultiCharEsc['W'].negate().freeze() diff --git a/pyxb/utils/xmlre.py b/pyxb/utils/xmlre.py index 6e9e50e0..0c0d04b6 100644 --- a/pyxb/utils/xmlre.py +++ b/pyxb/utils/xmlre.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2009-2013, Peter A. Bigot -# Copyright 2012, Jon Foster +# Copyright 2012,2018 Jon Foster +# Copyright 2018 Eurofins Digital Product Testing UK Ltd - https://www.eurofins-digitaltesting.com/ # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain a @@ -41,6 +42,10 @@ _log = logging.getLogger(__name__) +# =========================================================================== +# XML Schema character class parsing/conversion functions +# =========================================================================== + # AllEsc maps all the possible escape codes and wildcards in an XML schema # regular expression into the corresponding CodePointSet. _AllEsc = { } @@ -49,8 +54,8 @@ def _InitializeAllEsc (): """Set the values in _AllEsc without introducing C{k} and C{v} into the module.""" - _AllEsc.update({ six.u('.'): pyxb.utils.unicode.WildcardEsc }) - bs = six.unichr(0x5c) + _AllEsc[six.u('.')] = pyxb.utils.unicode.WildcardEsc + bs = b'\\'.decode('ascii') for k, v in six.iteritems(pyxb.utils.unicode.SingleCharEsc): _AllEsc[bs + six.text_type(k)] = v for k, v in six.iteritems(pyxb.utils.unicode.MultiCharEsc): @@ -154,7 +159,7 @@ class DashClass: elif ch == six.u(']'): # End break - elif ch == six.unichr(0x5c): # backslash + elif ch == b'\\'.decode('ascii'): cps, position = _MatchCharClassEsc(text, position) single_char = cps.asSingleCharacter() if single_char is not None: @@ -163,7 +168,7 @@ class DashClass: tokens.append(cps) elif ch == six.u('-'): # We need to distinguish between "-" and "\-". So we use - # DASH for a plain "-", and u"-" for a "\-". + # DASH for a plain "-", and "-" for a "\-". tokens.append(DASH) position = position + 1 else: @@ -198,6 +203,7 @@ class DashClass: elif isinstance(start, six.text_type): result_cps.add(ord(start)) else: + assert isinstance(start, pyxb.utils.unicode.CodePointSet) result_cps.extend(start) cur_token = cur_token + 1 @@ -229,7 +235,7 @@ def _MatchCharClassExpr(text, position): position = position + 1 if position >= len(text): raise RegularExpressionError(position, 'Missing character class expression') - negated = (text[position] == '^') + negated = (text[position] == six.u('^')) if negated: position = position + 1 @@ -249,32 +255,218 @@ def _MatchCharClassExpr(text, position): raise RegularExpressionError(position, "Expected ']' to end character class") return result_cps, position + 1 -def MaybeMatchCharacterClass (text, position): - """Attempt to match a U{character class expression - <http://www.w3.org/TR/xmlschema-2/#nt-charClassExpr>}. - @param text: The complete text of the regular expression being - translated +# =========================================================================== +# Utilities for Python's RE module +# =========================================================================== + +_python_re_escape_char_dict = { + six.u('\000'): b'\\000'.decode('ascii'), + six.u('\r'): b'\\r'.decode('ascii'), + six.u('\n'): b'\\n'.decode('ascii'), + six.u('.'): b'\\.'.decode('ascii'), + six.u('^'): b'\\^'.decode('ascii'), + six.u('$'): b'\\$'.decode('ascii'), + six.u('*'): b'\\*'.decode('ascii'), + six.u('+'): b'\\+'.decode('ascii'), + six.u('?'): b'\\?'.decode('ascii'), + six.u('{'): b'\\{'.decode('ascii'), + six.u('}'): b'\\}'.decode('ascii'), + b'\\'.decode('ascii'): b'\\\\'.decode('ascii'), + six.u('['): b'\\['.decode('ascii'), + six.u(']'): b'\\]'.decode('ascii'), + six.u('|'): b'\\|'.decode('ascii'), + six.u('('): b'\\('.decode('ascii'), + six.u(')'): b'\\)'.decode('ascii'), + six.u('-'): b'\\-'.decode('ascii'), # Needed inside [] blocks + } + +def _python_re_escape_char(char): + '''Escape characters that need it. Pass a single character only. + + Note: Python's re.escape() function it a little overeager to + escape everything. This only escapes things that need it.''' + return _python_re_escape_char_dict.get(char, char) + +def _AddQualifier(pattern, min_occurs, max_occurs): + assert isinstance(pattern, six.text_type) + assert len(pattern) >= 1 + assert isinstance(min_occurs, int) + assert min_occurs >= 0 + assert max_occurs is None or isinstance(max_occurs, int) + assert max_occurs is None or max_occurs >= 0 - @param position: The offset of the start of the potential - expression. + if min_occurs == 1 and max_occurs == 1: + pass + elif max_occurs is None: + if min_occurs == 0: + pattern += six.u('*') + elif min_occurs == 1: + pattern += six.u('+') + else: + pattern = six.u('%s{%d,}') % (pattern, min_occurs) + elif max_occurs == 1 and min_occurs == 0: + pattern += six.u('?') + elif max_occurs == 0: + pattern = six.u('') + elif min_occurs == max_occurs: + pattern = six.u('%s{%d}') % (pattern, min_occurs) + else: + pattern = six.u('%s{%d,%d}') % (pattern, min_occurs, max_occurs) + return pattern + + +# =========================================================================== +# Functions to parse a XSD regexp +# =========================================================================== + +# Char ::= [^.\?*+()|#x5B#x5D] +# This appears to be a bug in the spec - the text says {} are invalid, +# but the grammar doesn't. Excluding them because otherwise "a{4}" is +# ambiguous - does it only match the literal "aaaa" or only match the +# literal "a{4}"? (We only actually need to exclude "{" to make the +# grammar unambiguous, but this is cleaner and matches the standard's +# text). +_invalid_literal_chars = frozenset(b'.\\?*+()|[]{}'.decode('ascii')) + +def _MatchAtom(text, position): + '''Parses an "atom". + + This is either: + - "Char", a plain single character + - A bracketed "regExp" + - "charClassEsc" an escape code for either a single character or a range + of characters + - "WildcardEsc", the "." wildcard + - "charClassExpr", a character class using the [] syntax + + Preconditions: Not at end of string. + If the parsing fails, throws a RegularExpressionError. + Returns a tuple with the Python regex that matches the atom, and the new + position. + Postconditions: None. + ''' + assert position < len(text) + start_position = position + ch = text[position] + if ch not in _invalid_literal_chars: + atom_pattern = _python_re_escape_char(ch) + position = position + 1 + elif ch == six.u('('): + atom_pattern, position = _MatchSubRegex(text, position + 1) + if position >= len(text) or text[position] != six.u(')'): + raise RegularExpressionError(start_position, "Unmatched bracket") + position = position + 1 + elif ch == b'\\'.decode('ascii'): + char_class, position = _MatchCharClassEsc(text, position) + single_char = char_class.asSingleCharacter() + if single_char is not None: + # E.g. '\\\\' isn't really a char range. + atom_pattern = _python_re_escape_char(single_char) + else: + atom_pattern = char_class.asPattern() + elif ch == six.u('.'): + atom_pattern = pyxb.utils.unicode.WildcardEsc.asPattern() + position = position + 1 + elif ch == six.u('['): + char_class, position = _MatchCharClassExpr(text, position) + atom_pattern = char_class.asPattern() + else: + raise RegularExpressionError(position, "Invalid character") + return atom_pattern, position + +_quantifier_curlybrace_re = re.compile( + b'\\{(?:' + b'(?P<exact_occurs>[0-9]+)' # {3} style + b'|' + b'(?:(?P<min_occurs>[0-9]+),(?P<max_occurs>[0-9]+)?)' # {3,4} or {3,} + b')\\}'.decode('ascii')) + +def _MatchQuantifier(text, position): + '''Tries to parse a "quantifier", if present. + If not, just returns the default quantifier of {1,1}. + + Preconditions: None. + If there's a "{" character indicating the start of a quantifier, but + parsing it fails, throws a RegularExpressionError. Will not throw if + there's no quantifier at all. + Returns a tuple with the min and max occurs, and the new position. Max + occurs can be None for unlimited. + Postconditions: None. + ''' - @return: C{None} if C{position} does not begin a character class - expression; otherwise a pair C{(cps, p)} where C{cps} is a - L{pyxb.utils.unicode.CodePointSet} containing the code points associated with - the property, and C{p} is the text offset immediately following - the closing brace.""" - if position >= len(text): - return None - c = text[position] - np = position + 1 - if '.' == c: - return (pyxb.utils.unicode.WildcardEsc, np) - if '[' == c: - return _MatchCharClassExpr(text, position) - if '\\' == c: - return _MatchCharClassEsc(text, position) - return None + min_occurs = 1 + max_occurs = 1 + if position < len(text): + ch = text[position] + if ch == six.u('?'): + min_occurs = 0 + position = position + 1 + elif ch == six.u('*'): + min_occurs = 0 + max_occurs = None + position = position + 1 + elif ch == six.u('+'): + max_occurs = None + position = position + 1 + elif ch == six.u('{'): + mo = _quantifier_curlybrace_re.match(text, position) + if not mo: + raise RegularExpressionError(position, "Cannot parse quantifier starting '{')") + exact_occurs = mo.group('exact_occurs') + if exact_occurs is not None: + min_occurs = max_occurs = int(exact_occurs, 10) + else: + min_occurs = int(mo.group('min_occurs'), 10) + max_occurs = mo.group('max_occurs') + if max_occurs is not None: + max_occurs = int(max_occurs, 10) + position = mo.end() + return min_occurs, max_occurs, position + +def _MatchBranch(text, position): + '''Parses a "branch". This is a series of "piece"s. It doesn't contain + the "|" character (unless it's bracketed or escaped). + + Each "piece" is an "atom" with an optional "qualifier". + + Preconditions: None + If the parsing fails, throws a RegularExpressionError. + Returns a tuple with the (possibly empty) Python regex that matches the + branch and the new position. + Postconditions: At end of string, or next character is '|' or ')'. + ''' + pieces = [] + while position < len(text) and text[position] != six.u('|') and text[position] != six.u(')'): + atom_pattern, position = _MatchAtom(text, position) + min_occurs, max_occurs, position = _MatchQuantifier(text, position) + pieces.append(_AddQualifier(atom_pattern, min_occurs, max_occurs)) + + pattern = six.u('').join(pieces) + return pattern, position + +def _MatchSubRegex(text, position): + '''Parses a "regExp". This is one or more "branch"es, separated by "|" + characters. + + Preconditions: None + If the parsing fails, throws a RegularExpressionError. + Returns a tuple with the Python regex that matches the XSD regex and the + new position. + Postconditions: At end of string, or next character is ')'. + ''' + branches = [] + new_branch, position = _MatchBranch(text, position) + branches.append(new_branch) + while position < len(text) and text[position] == six.u('|'): + new_branch, position = _MatchBranch(text, position + 1) + branches.append(new_branch) + pattern = six.u('(?:%s)') % (six.u('|').join(branches),) + return pattern, position + +# =========================================================================== +# Main XSD-to-Python regex conversion function +# =========================================================================== def XMLToPython (pattern): """Convert the given pattern to the format required for Python @@ -287,23 +479,7 @@ def XMLToPython (pattern): @return: A Unicode string specifying a Python regular expression that matches the same language as C{pattern}.""" assert isinstance(pattern, six.text_type) - new_pattern_elts = [] - new_pattern_elts.append('^(') - position = 0 - while position < len(pattern): - cg = MaybeMatchCharacterClass(pattern, position) - if cg is None: - ch = pattern[position] - if ch == six.u('^') or ch == six.u('$'): - # These characters have no special meaning in XSD. But they - # match start and end of string in Python, so they have to - # be escaped. - new_pattern_elts.append(six.unichr(0x5c) + ch) - else: - new_pattern_elts.append(ch) - position += 1 - else: - (cps, position) = cg - new_pattern_elts.append(cps.asPattern()) - new_pattern_elts.append(')$') - return ''.join(new_pattern_elts) + py_pattern, position = _MatchSubRegex(pattern, 0) + if position != len(pattern): + raise RegularExpressionError() + return six.u("^%s$") % (py_pattern,) diff --git a/tests/utils/test-xmlre.py b/tests/utils/test-xmlre.py index 88af01cb..3923119c 100644 --- a/tests/utils/test-xmlre.py +++ b/tests/utils/test-xmlre.py @@ -4,7 +4,7 @@ if __name__ == '__main__': logging.basicConfig() _log = logging.getLogger(__name__) -from pyxb.utils import unicode, xmlre +from pyxb.utils import unicode, xmlre, six import re import unittest @@ -25,88 +25,85 @@ def assertNoMatch(self, xml_pattern, value): mo = compiled.match(value) self.assertTrue(mo is None, 'XML re %r Python %r should not match %r' % (xml_pattern, py_pattern, value)) - def testRangeErrors (self): - self.assertTrue(xmlre.MaybeMatchCharacterClass('', 1) is None) - def testWildcardEscape (self): - (charset, position) = xmlre.MaybeMatchCharacterClass('.', 0) - self.assertEqual(charset, unicode.WildcardEsc) + (charset, position) = xmlre._MatchAtom('.', 0) + self.assertEqual(charset, unicode.WildcardEsc.asPattern()) self.assertEqual(position, 1) def testSingleCharEscapes (self): # 17 chars recognized as escapes self.assertEqual(len(unicode.SingleCharEsc), 17) - (charset, position) = xmlre.MaybeMatchCharacterClass(r'\t', 0) - self.assertEqual(charset.asTuples(), [ (9, 9) ]) + (charset, position) = xmlre._MatchAtom(r'\t', 0) + self.assertEqual(charset, '\t') self.assertEqual(2, position) - (charset, position) = xmlre.MaybeMatchCharacterClass(r'\?', 0) - self.assertEqual(charset.asTuples(), [ (ord('?'), ord('?')) ]) + (charset, position) = xmlre._MatchAtom(r'\?', 0) + self.assertEqual(charset, '\\?') self.assertEqual(2, position) - (charset, position) = xmlre.MaybeMatchCharacterClass(r'\\', 0) - self.assertEqual(charset.asTuples(), [ (ord('\\'), ord('\\')) ]) + (charset, position) = xmlre._MatchAtom(r'\\', 0) + self.assertEqual(charset, '\\\\') self.assertEqual(2, position) def testMultiCharEscapes (self): # 5*2 chars recognized as escapes self.assertEqual(len(unicode.MultiCharEsc), 10) - (charset, position) = xmlre.MaybeMatchCharacterClass(r'\s', 0) - self.assertEqual(charset.asTuples(), [ (9, 10), (13, 13), (32, 32) ]) + (charset, position) = xmlre._MatchAtom(r'\s', 0) + self.assertEqual(charset, '[\t-\n\r ]') self.assertEqual(2, position) def testMatchCharProperty (self): - self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchCharClassEsc, "\pL", 0) - self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchCharClassEsc, "\p{L", 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, "\pL", 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, "\p{L", 0) text = "\p{L}" - (charset, position) = xmlre._MatchCharClassEsc(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset, unicode.PropertyMap['L']) + self.assertEqual(charset, unicode.PropertyMap['L'].asPattern()) text = "\p{IsCyrillic}" - (charset, position) = xmlre._MatchCharClassEsc(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset, unicode.BlockMap['Cyrillic']) + self.assertEqual(charset, unicode.BlockMap['Cyrillic'].asPattern()) def testCharProperty (self): text = r'\p{D}' - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, text, 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, text, 0) text = r'\P{D}' - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, text, 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, text, 0) text = r'\p{N}' - (charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset, unicode.PropertyMap['N']) + self.assertEqual(charset, unicode.PropertyMap['N'].asPattern()) text = r'\P{N}' - (charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset.negate(), unicode.PropertyMap['N']) + self.assertEqual(charset, unicode.PropertyMap['N'].negate().asPattern()) text = r'\p{Sm}' - (charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset, unicode.PropertyMap['Sm']) + self.assertEqual(charset, unicode.PropertyMap['Sm'].asPattern()) def testCharBlock (self): text = r'\p{IsArrows}' - (charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset, unicode.BlockMap['Arrows']) + self.assertEqual(charset, unicode.BlockMap['Arrows'].asPattern()) text = r'\P{IsArrows}' - (charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + (charset, position) = xmlre._MatchAtom(text, 0) self.assertEqual(position, len(text)) - self.assertEqual(charset.negate(), unicode.BlockMap['Arrows']) + self.assertEqual(charset, unicode.BlockMap['Arrows'].negate().asPattern()) text = r'\p{IsWelsh}' - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, text, 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, text, 0) text = r'\P{IsWelsh}' - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, text, 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, text, 0) def testCharGroup (self): - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, '[]', 0) - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, '[A--]', 0) - self.assertRaises(xmlre.RegularExpressionError, xmlre.MaybeMatchCharacterClass, '[A--]', 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, '[]', 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, '[A--]', 0) + self.assertRaises(xmlre.RegularExpressionError, xmlre._MatchAtom, '[A--]', 0) text = r'[A-Z]' - #(charset, position) = xmlre.MaybeMatchCharacterClass(text, 0) + #(charset, position) = xmlre._MatchAtom(text, 0) #self.assertEqual(position, len(text)) #self.assertEqual(charset, unicode.CodePointSet((ord('A'), ord('Z')))) @@ -178,10 +175,10 @@ def testMatchCharClassExpr (self): self.assertEqual(charset, expected) def testXMLToPython (self): - self.assertEqual(r'^(123)$', xmlre.XMLToPython('123')) + self.assertEqual(r'^(?:123)$', xmlre.XMLToPython('123')) # Note that single-char escapes in the expression are - # converted to character classes. - self.assertEqual(r'^(Why[ ]not[?])$', xmlre.XMLToPython(r'Why[ ]not\?')) + # no longer converted to character classes. + self.assertEqual('^(?:Why[ ]not\\?)$', xmlre.XMLToPython(r'Why[ ]not\?')) def testRegularExpressions (self): text = '[\i-[:]][\c-[:]]*' From f945023582012d6e44707983e7828138d4183cb5 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Fri, 23 Feb 2018 17:21:25 +0000 Subject: [PATCH 02/10] Make "setup.py bdist_wheel" work Use setuptools instead of distutils, for wheel support. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a19b0bb..48b25590 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ import datetime import logging -from distutils.core import setup, Command +from setuptools import setup, Command # Stupid little command to automatically update the version number # where it needs to be updated. From fd9c6806f9343e66977cb51bc230603b3b0bc285 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Fri, 23 Feb 2018 17:23:28 +0000 Subject: [PATCH 03/10] Ignore build files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 82e1c141..126f38d6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ *.wxs test.log /build/ +/dist/ +/PyXB.egg-info/ \ No newline at end of file From b48a140588c1d3a7c4c8341b874b1c4ab54f61d2 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Mon, 26 Mar 2018 15:25:59 +0100 Subject: [PATCH 04/10] Make version number comply with PEP440 This fixes a warning when running setup.py, due to the version number having an invalid format. --- README.txt | 2 +- doc/conf.py | 4 ++-- pyxb/__init__.py | 2 +- setup.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.txt b/README.txt index 8d58e3ed..5985c85a 100644 --- a/README.txt +++ b/README.txt @@ -1,5 +1,5 @@ PyXB -- Python W3C XML Schema Bindings -Version 1.2.7-DEV +Version 1.2.7.dev1 The source releases includes pre-built bundles for common XML namespaces, assorted web service namespaces, and SAML. A bundle with over 75 namespaces diff --git a/doc/conf.py b/doc/conf.py index 90bf8d7a..82f36d1b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -43,7 +43,7 @@ # General information about the project. project = 'PyXB' -copyright = '2009-2017, Peter A. Bigot' +copyright = '2009-2018, Peter A. Bigot' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.2' # The full version, including alpha/beta/rc tags. -release = '1.2.7-DEV' +release = '1.2.7.dev1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pyxb/__init__.py b/pyxb/__init__.py index 1773f16d..33ea7549 100644 --- a/pyxb/__init__.py +++ b/pyxb/__init__.py @@ -61,7 +61,7 @@ def __init__ (self, *args, **kw): if issubclass(self.__class__.mro()[-2], ( list, dict )): super(cscRoot, self).__init__(*args) -__version__ = '1.2.7-DEV' +__version__ = '1.2.7.dev1' """The version of PyXB""" __url__ = 'http://pyxb.sourceforge.net' diff --git a/setup.py b/setup.py index 48b25590..0b52404b 100755 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ from __future__ import print_function import sys -# The current version of the system. Format is #.#.#[-DEV]. -version = '1.2.7-DEV' +# The current version of the system. Format is '#.#.#[.dev#]'. +version = '1.2.7.dev1' # Require Python 2.6 or higher or Python 3.1 or higher if (sys.version_info[:2] < (2, 6)) or ((sys.version_info[0] == 3) and sys.version_info[:2] < (3, 1)): From 9298678691eb1ecf3c747d0c30dbf15539f90312 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Mon, 26 Mar 2018 15:30:29 +0100 Subject: [PATCH 05/10] Remove Python 2.5 compatibility code + privacy fix Really old versions of Python are no longer supported by PyXB anyway, so there's no need to keep complicated code just to support them. Also use UUID4 rather than UUID1 for generating UUIDs, because there are privacy issues with people's MAC address being embedded in UUIDs in documents. --- pyxb/utils/utility.py | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/pyxb/utils/utility.py b/pyxb/utils/utility.py index 549f8970..ea7f9cfb 100644 --- a/pyxb/utils/utility.py +++ b/pyxb/utils/utility.py @@ -18,6 +18,8 @@ import re import os import errno +import uuid +import hashlib import pyxb from pyxb.utils.six.moves.urllib import parse as urlparse import time @@ -829,15 +831,6 @@ def OpenOrCreate (file_name, tag=None, preserve_contents=False): fp.seek(2) # os.SEEK_END return fp -# hashlib didn't show up until 2.5, and sha is deprecated in 2.6. -__Hasher = None -try: - import hashlib - __Hasher = hashlib.sha1 -except ImportError: - import sha - __Hasher = sha.new - def HashForText (text): """Calculate a cryptographic hash of the given string. @@ -850,26 +843,7 @@ def HashForText (text): """ if isinstance(text, six.text_type): text = text.encode('utf-8') - return __Hasher(text).hexdigest() - -# uuid didn't show up until 2.5 -__HaveUUID = False -try: - import uuid - __HaveUUID = True -except ImportError: - import random -def _NewUUIDString (): - """Obtain a UUID using the best available method. On a version of - python that does not incorporate the C{uuid} class, this creates a - string combining the current date and time (to the second) with a - random number. - - @rtype: C{str} - """ - if __HaveUUID: - return uuid.uuid1().urn - return '%s:%08.8x' % (time.strftime('%Y%m%d%H%M%S'), random.randint(0, 0xFFFFFFFF)) + return hashlib.sha1(text).hexdigest() class UniqueIdentifier (object): """Records a unique identifier, generally associated with a @@ -908,7 +882,7 @@ def __setstate__ (self, state): # Singleton-like def __new__ (cls, *args): if 0 == len(args): - uid = _NewUUIDString() + uid = uuid.uuid4().urn else: uid = args[0] if isinstance(uid, UniqueIdentifier): From 265d5c0c0071231d2d47c54cd6b1e6eea62f87fb Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Mon, 26 Mar 2018 15:36:31 +0100 Subject: [PATCH 06/10] Make binding generation reproducible Make it posible to configure pyxbgen to generate bindings in a reproducible way. "Reproducible" means that the same input results in exactly the same output, byte-for-byte, without depending on: * the time the bindings were built * the path to the working directory they're buit in * the MAC address of the build PC (embedded in a UUID) * newly generated UUIDs (guaranteed to be different every time) * the iteration order of Python dicts/sets * the path separator (e.g. "/") used by the OS that the bindings are built on. Before this commit, the PyXB bindings changed every time they were built becasue they depended on all of the above. After this commit, you can pass options to pyxbgen to get reproducible bindings. The new options are: * --strip-file-paths - doesn't store paths to XSD files in bindings. * --no-timestamp - doesn't write build timestamp and Python version in comments in bindings. * --generation-uid= - specifies the Generation UID rather than randomly generating one. There are two reasons why people want this: 1) If bindings are committed to version control, then when bindings are rebuilt you want to see what actually changed and not see loads of spurious differences. 2) If bindings are not committed to version control but are built by some build system, then careful development teams want developers to be using the exact same bindings as each other and as the build system is building. --- pyxb/binding/generate.py | 170 +++++++++++++++++++++++++++++++-------- pyxb/utils/utility.py | 21 +++++ 2 files changed, 156 insertions(+), 35 deletions(-) diff --git a/pyxb/binding/generate.py b/pyxb/binding/generate.py index 87f6d5f2..37f18332 100644 --- a/pyxb/binding/generate.py +++ b/pyxb/binding/generate.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2009-2013, Peter A. Bigot +# Copyright 2018 Eurofins Digital Product Testing UK Ltd - https://www.eurofins-digitaltesting.com/ # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain a @@ -33,6 +34,9 @@ _log = logging.getLogger(__name__) +_StripFilePaths = False +_NoTimestamp = False + def PrefixModule (value, text=None): if text is None: text = value.__name__ @@ -42,6 +46,22 @@ def PrefixModule (value, text=None): return 'pyxb.binding.facets.%s' % (text,) raise ValueError('No standard name for module of value', value) +def _LocationRepr(loc): + '''Return the representation of a Location object, as a string. + + This will remove the directory part of file paths, if requested. + + There are two reasons for this: It hides potentially-sensitive file paths + (e.g. usernames if building in My Documents on Windows or in a user's home + directory on Unix systems). It also helps ensure that builds are + reproducible - if different users build the same bindings from the same + source files they should end up with the same binding Python files, even if + they build in different directories or on different OSs. + ''' + if _StripFilePaths and loc is not None: + loc = loc.withFilePathsRemoved() + return repr2to3(loc) + class ReferenceLiteral (object): """Base class for something that requires fairly complex activity in order to generate its literal value.""" @@ -193,7 +213,12 @@ def __init__ (self, **kw): def pythonLiteral (value, **kw): # For dictionaries, apply translation to all values (not keys) if isinstance(value, six.dictionary_type): - return ', '.join([ '%s=%s' % (k, pythonLiteral(v, **kw)) for (k, v) in six.iteritems(value) ]) + entries = [ (k, pythonLiteral(v, **kw)) for (k, v) in six.iteritems(value) ] + # It's important to sort the entries here, so we reproducibly generate + # the same code for the same input. It really doesn't matter *how* + # they're sorted. + entries.sort() + return ', '.join(('%s=%s' % (k, v)) for (k, v) in entries) # For lists, apply translation to all members if isinstance(value, six.list_type): @@ -306,7 +331,7 @@ def transitionSortKey (xit): for cc in sorted_counter_conditions: cc_id = 'cc_%u' % (len(counter_map),) counter_map[cc] = cc_id - au_src.append(' %s = fac.CounterCondition(min=%s, max=%s, metadata=%r)' % (cc_id, repr2to3(cc.min), repr2to3(cc.max), cc.metadata._location())) + au_src.append(' %s = fac.CounterCondition(min=%s, max=%s, metadata=%s)' % (cc_id, repr2to3(cc.min), repr2to3(cc.max), _LocationRepr(cc.metadata._location()))) au_src.append(' counters.add(%s)' % (cc_id,)) state_map = {} au_src.append(' states = []') @@ -325,14 +350,14 @@ def transitionSortKey (xit): for ui in sorted(st.finalUpdate, key=updateInstructionSortKey): au_src.append(' final_update.add(fac.UpdateInstruction(%s, %r))' % (counter_map[ui.counterCondition], ui.doIncrement)) if isinstance(st.symbol, xs.structures.ModelGroup): - au_src.append(' symbol = %r' % (st.symbol._location(),)) + au_src.append(' symbol = %s' % (_LocationRepr(st.symbol._location()),)) else: (particle, symbol) = st.symbol if isinstance(symbol, xs.structures.Wildcard): - au_src.append(templates.replaceInText(' symbol = pyxb.binding.content.WildcardUse(%{wildcard}, %{location})', wildcard=binding_module.literal(symbol, **kw), location=repr2to3(particle._location()))) + au_src.append(templates.replaceInText(' symbol = pyxb.binding.content.WildcardUse(%{wildcard}, %{location})', wildcard=binding_module.literal(symbol, **kw), location=_LocationRepr(particle._location()))) elif isinstance(symbol, xs.structures.ElementDeclaration): binding_module.importForDeclaration(symbol) - au_src.append(templates.replaceInText(' symbol = pyxb.binding.content.ElementUse(%{ctd}._UseForTag(%{field_tag}), %{location})', field_tag=binding_module.literal(symbol.expandedName(), **kw), location=repr2to3(particle._location()), **template_map)) + au_src.append(templates.replaceInText(' symbol = pyxb.binding.content.ElementUse(%{ctd}._UseForTag(%{field_tag}), %{location})', field_tag=binding_module.literal(symbol.expandedName(), **kw), location=_LocationRepr(particle._location()), **template_map)) au_src.append(' %s = fac.State(symbol, is_initial=%r, final_update=final_update, is_unordered_catenation=%r)' % (st_id, st.isInitial, st.isUnorderedCatenation)) if st.subAutomata is not None: au_src.append(' %s._set_subAutomata(*sub_automata)' % (st_id,)) @@ -406,8 +431,9 @@ def GenerateFacets (td, generator, **kw): if isinstance(fi, facets.CF_enumeration): argset['enum_prefix'] = fi.enumPrefix() facet_var = ReferenceFacetMember(type_definition=td, facet_class=fc, **kw) - outf.write("%s = %s(%s)\n" % binding_module.literal( (facet_var, fc, argset ), **kw)) - facet_instances.append(binding_module.literal(facet_var, **kw)) + facet_code = [] + facet_instances.append((binding_module.literal(facet_var, **kw), facet_code)) + facet_code.append("%s = %s(%s)\n" % binding_module.literal( (facet_var, fc, argset ), **kw)) if (fi is not None) and is_collection: for i in six.iteritems(fi): if isinstance(i, facets._EnumerationElement): @@ -417,13 +443,20 @@ def GenerateFacets (td, generator, **kw): enum_config = '%s.addEnumeration(unicode_value=%s, tag=%s)' % binding_module.literal( ( facet_var, i.unicodeValue(), i.tag() ), **kw) if gen_enum_tag and (i.tag() is not None): enum_member = ReferenceEnumerationMember(type_definition=td, facet_instance=fi, enumeration_element=i, **kw) - outf.write("%s = %s\n" % (binding_module.literal(enum_member, **kw), enum_config)) + facet_code.append("%s = %s\n" % (binding_module.literal(enum_member, **kw), enum_config)) if fi.enumPrefix() is not None: - outf.write("%s_%s = %s\n" % (fi.enumPrefix(), i.tag(), binding_module.literal(enum_member, **kw))) + facet_code.append("%s_%s = %s\n" % (fi.enumPrefix(), i.tag(), binding_module.literal(enum_member, **kw))) else: - outf.write("%s\n" % (enum_config,)) + facet_code.append("%s\n" % (enum_config,)) if isinstance(i, facets._PatternElement): - outf.write("%s.addPattern(pattern=%s)\n" % binding_module.literal( (facet_var, i.pattern ), **kw)) + facet_code.append("%s.addPattern(pattern=%s)\n" % binding_module.literal( (facet_var, i.pattern ), **kw)) + # It's important to sort the facets here, so we reproducibly generate + # the same code for the same input. It really doesn't matter *how* + # they're sorted. + facet_instances.sort() + for _, facet_code in facet_instances: + for s in facet_code: + outf.write(s) if gen_enum_tag and (xs.structures.SimpleTypeDefinition.VARIETY_union == td.variety()): # If the union has enumerations of its own, there's no need to # inherit anything, because they supersede anything implicitly @@ -444,9 +477,9 @@ def GenerateFacets (td, generator, **kw): outf.write("%-50s%s\n" % ('%s = %s' % binding_module.literal( (enum_member, i.unicodeValue()) ), '# originally %s.%s' % (binding_module.literal(etd), i.tag()))) if 2 <= len(facet_instances): - map_args = ",\n ".join(facet_instances) + map_args = ",\n ".join(facet_var for facet_var, _ in facet_instances) else: - map_args = ','.join(facet_instances) + map_args = ','.join(facet_var for facet_var, _ in facet_instances) outf.write("%s._InitializeFacetMap(%s)\n" % (binding_module.literal(td, **kw), map_args)) def _VCAppendAuxInit (vc_source, aux_init, binding_module, kw): @@ -497,7 +530,7 @@ def GenerateSTD (std, generator): else: template_map['qname'] = '[anonymous]' template_map['namespaceReference'] = binding_module.literal(std.bindingNamespace(), **kw) - template_map['xsd_location'] = repr2to3(std._location()) + template_map['xsd_location'] = _LocationRepr(std._location()) if std.annotation() is not None: template_map['documentation'] = std.annotation().asDocString() template_map['documentation_expr'] = binding_module.literal(std.annotation().text()) @@ -565,7 +598,7 @@ class %{std} (pyxb.binding.basis.STD_union): def elementDeclarationMap (ed, binding_module, **kw): template_map = { } template_map['qname'] = six.text_type(ed.expandedName()) - template_map['decl_location'] = repr2to3(ed._location()) + template_map['decl_location'] = _LocationRepr(ed._location()) template_map['namespaceReference'] = binding_module.literal(ed.bindingNamespace(), **kw) if (ed.SCOPE_global == ed.scope()): binding_name = template_map['class'] = binding_module.literal(ed, **kw) @@ -871,7 +904,7 @@ def GenerateCTD (ctd, generator, **kw): template_map['qname'] = six.text_type(ctd.expandedName()) else: template_map['qname'] = '[anonymous]' - template_map['xsd_location'] = repr2to3(ctd._location()) + template_map['xsd_location'] = _LocationRepr(ctd._location()) template_map['simple_base_type'] = binding_module.literal(None, **kw) template_map['contentTypeTag'] = content_type_tag template_map['is_abstract'] = repr2to3(not not ctd.abstract()) @@ -1024,8 +1057,8 @@ class %{ctd} (%{superclass}): assert ad.typeDefinition() is not None au_map['attr_type'] = binding_module.literal(ad.typeDefinition(), in_class=True, **kw) - au_map['decl_location'] = repr2to3(ad._location()) - au_map['use_location'] = repr2to3(au._location()) + au_map['decl_location'] = _LocationRepr(ad._location()) + au_map['use_location'] = _LocationRepr(au._location()) vc_source = ad if au.valueConstraint() is not None: @@ -1164,8 +1197,13 @@ def __init__ (self, binding_module, **kw): self.__postscript = [] self.__templateMap = kw.copy() encoding = kw.get('encoding', pyxb._OutputEncoding) + # It's important to replace the OS-dependent path separator with + # a fixed value here, so we reproducibly generate the same code + # for the same input regardless of what OS this is run on. + # It really doesn't matter what separator we use, so long as it's + # consistent, so we use '/'. self.__templateMap.update({ 'date' : str(datetime.datetime.now()), - 'filePath' : self.__bindingFilePath, + 'filePath' : self.__bindingFilePath.replace(os.path.sep, '/'), 'coding' : encoding, 'binding_module' : binding_module, 'binding_tag' : binding_module.bindingTag(), @@ -1174,11 +1212,19 @@ def __init__ (self, binding_module, **kw): 'pythonVersion' : '.'.join(map(str, sys.version_info))}) self.__stringIO = io.StringIO() if self.__bindingFile: - prefacet = self.expand('''# %{filePath} + if _NoTimestamp: + prefacet_template = '''# %{filePath} +# -*- coding: %{coding} -*- +# PyXB bindings for %{binding_tag} +# Generated by PyXB version %{pyxbVersion} +%{binding_preface}''' + else: + prefacet_template = '''# %{filePath} # -*- coding: %{coding} -*- # PyXB bindings for %{binding_tag} # Generated %{date} by PyXB version %{pyxbVersion} using Python %{pythonVersion} -%{binding_preface}''', binding_preface=binding_module.bindingPreface()) +%{binding_preface}''' + prefacet = self.expand(prefacet_template, binding_preface=binding_module.bindingPreface()) self.__bindingFile.write(prefacet.encode(encoding)) self.__bindingFile.flush() @@ -1362,6 +1408,10 @@ def moduleContents (self): aux_imports.append('import %s as %s' % (mr.modulePath(), as_path)) else: aux_imports.append('import %s' % (mr.modulePath(),)) + # It's important to sort the imports here, so we reproducibly generate + # the same code for the same input. It really doesn't matter *how* + # they're sorted. + aux_imports.sort() template_map['aux_imports'] = "\n".join(aux_imports) template_map['namespace_decls'] = "\n".join(self.__namespaceDeclarations) template_map['module_uid'] = self.moduleUID() @@ -2314,6 +2364,44 @@ def setLoggingConfigFile (self, logging_config_file): self.__loggingConfigFile = logging_config_file __loggingConfigFile = None + def stripFilePaths (self): + """Don't store schema file paths in the generated bindings, only store + filenames. This hides potentially-sensitive file paths (e.g. + usernames), and helps ensure that builds are reproducible. + + @rtype: C{bool}""" + return self.__stripFilePaths + def setStripFilePaths (self, value): + self.__stripFilePaths = value + global _StripFilePaths + _StripFilePaths = value + __stripFilePaths = None + + def noTimestamp (self): + """Don't store timestamps and Python version in comments in the + generated bindings. This helps ensure that builds are reproducible. + + @rtype: C{bool}""" + return self.__stripFilePaths + def setNoTimestamp (self, value): + self.__noTimestamp = value + global _NoTimestamp + _NoTimestamp = value + __noTimestamp = None + + def generationUID (self): + """A unique identifier associated with this Generator instance. + + This is an instance of L{pyxb.utils.utility.UniqueIdentifier}. + Its associated objects are + L{pyxb.namespace.archive._SchemaOrigin} instances, which + identify schema that contribute to the definition of a + namespace.""" + return self.__generationUID + def setGenerationUID (self, value): + self.__generationUID = pyxb.utils.utility.UniqueIdentifier(value) + __generationUID = None + def __init__ (self, *args, **kw): """Create a configuration to be used for generating bindings. @@ -2344,6 +2432,8 @@ def __init__ (self, *args, **kw): @keyword generate_to_files: Sets L{generateToFiles} @keyword uri_content_archive_directory: Invokes L{setUriContentArchiveDirectory} @keyword logging_config_file: Invokes L{setLoggingConfigFile} + @keyword strip_file_paths: Invokes L{setStripFilePaths} + @keyword no_timestamp: Invokes L{setNoTimestamp} """ argv = kw.get('argv') if argv is not None: @@ -2373,14 +2463,20 @@ def __init__ (self, *args, **kw): self.__generateToFiles = kw.get('generate_to_files', True) self.__uriContentArchiveDirectory = kw.get('uri_content_archive_directory') self.__loggingConfigFile = kw.get('logging_config_file') + self.__stripFilePaths = kw.get('strip_file_paths', False) + self.__noTimestamp = kw.get('no_timestamp', False) self.__unnamedModulePaths = set() + generationUID = kw.get('generation_uid', None) + if generationUID is None: + self.__generationUID = pyxb.utils.utility.UniqueIdentifier() + else: + self.__generationUID = pyxb.utils.utility.UniqueIdentifier(generationUID) + if argv is not None: self.applyOptionValues(*self.optionParser().parse_args(argv)) [ self.addSchemaLocation(_a) for _a in args ] - self.__generationUID = pyxb.utils.utility.UniqueIdentifier() - pyxb.namespace.XML.validateComponentModel() __stripSpaces_re = re.compile('\s\s\s+') @@ -2405,7 +2501,10 @@ def __stripSpaces (self, string): ('allow_builtin_generation', setAllowBuiltinGeneration), ('allow_absent_module', setAllowAbsentModule), ('uri_content_archive_directory', setUriContentArchiveDirectory), - ('logging_config_file', setLoggingConfigFile) + ('logging_config_file', setLoggingConfigFile), + ('strip_file_paths', setStripFilePaths), + ('no_timestamp', setNoTimestamp), + ('generation_uid', setGenerationUID) ) def applyOptionValues (self, options, args=None): for (tag, method) in self.__OptionSetters: @@ -2428,17 +2527,6 @@ def setFromCommandLine (self, argv=None): self.applyOptionValues(options, args) return self - def generationUID (self): - """A unique identifier associated with this Generator instance. - - This is an instance of L{pyxb.utils.utility.UniqueIdentifier}. - Its associated objects are - L{pyxb.namespace.archive._SchemaOrigin} instances, which - identify schema that contribute to the definition of a - namespace.""" - return self.__generationUID - __generationUID = None - def optionParser (self, reset=False): """Return an C{optparse.OptionParser} instance tied to this configuration. @@ -2519,6 +2607,18 @@ def optionParser (self, reset=False): help=self.__stripSpaces(self.validateChanges.__doc__ + ' This option turns off validation.')) parser.add_option_group(group) + group = optparse.OptionGroup(parser, 'Reproducible builds', "Options to help ensure that bindings are always generated the same.") + group.add_option('--strip-file-paths', + action="store_true", dest='strip_file_paths', + help=self.__stripSpaces(self.stripFilePaths.__doc__)) + group.add_option('--no-timestamp', + action="store_true", dest='no_timestamp', + help=self.__stripSpaces(self.noTimestamp.__doc__)) + group.add_option('--generation-uid', + dest='generation_uid', + help='Unique ID for this generation run. A urn:uuid: value. Defaults to a random value.') + parser.add_option_group(group) + group = optparse.OptionGroup(parser, 'Miscellaneous Options', "Anything else.") group.add_option('--logging-config-file', metavar="FILE", help=self.__stripSpaces(self.loggingConfigFile.__doc__)) diff --git a/pyxb/utils/utility.py b/pyxb/utils/utility.py index ea7f9cfb..a84e3919 100644 --- a/pyxb/utils/utility.py +++ b/pyxb/utils/utility.py @@ -1274,6 +1274,27 @@ def __repr__ (self): ctor = '%s.%s' % (t.__module__, t.__name__) return '%s(%s, %r, %r)' % (ctor, repr2to3(self.__locationBase), self.__lineNumber, self.__columnNumber) + def withFilePathsRemoved(self): + '''Return a version of this Locator with any file paths removed. + Just the file name remains. + + If this Locator points to a HTTP or HTTPS URL, it is returned unchanged. + + This is used when generating bindings. There are two reasons for this: + It hides potentially-sensitive file paths (e.g. usernames if building + in My Documents on Windows or in a user's home directory on Unix + systems). It also helps ensure that builds are reproducible - if + different users build the same bindings from the same source files they + should end up with the same binding Python files, even if they build + in different directories or on different OSs. + ''' + proto = self.__locationBase.split(':', 1)[0] + if proto in ('http', 'https'): + return self + filename = self.__locationBase.rsplit('/', 1)[-1] + return type(self)(filename, self.__lineNumber, self.__columnNumber) + + class Locatable_mixin (pyxb.cscRoot): __location = None From 6d68f6cdbda9023f4643f003131d7a3e9461b488 Mon Sep 17 00:00:00 2001 From: Jon Foster <jon@jon-foster.co.uk> Date: Mon, 26 Mar 2018 15:36:44 +0100 Subject: [PATCH 07/10] Update copyright notice --- NOTICE | 1 + 1 file changed, 1 insertion(+) diff --git a/NOTICE b/NOTICE index 826a29e7..1a7c2e45 100644 --- a/NOTICE +++ b/NOTICE @@ -20,6 +20,7 @@ PyXB includes patches from the following Contributors: Michael van der Westhuizen Jon Foster + Eurofins Digital Testing (https://www.eurofins-digitaltesting.com/) For details on these patches see the git software configuration logs. From de55ec52ad93b9873db2c5fbcea7cdfa338fa5d2 Mon Sep 17 00:00:00 2001 From: Oleg Broytman <phd@phdru.name> Date: Tue, 26 Feb 2019 22:21:29 +0300 Subject: [PATCH 08/10] Fix simpleTypeDefinition.XsdSuperType Do not return intermediate internal classes like `_PyXBDateOnly_base`. --- pyxb/binding/basis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyxb/binding/basis.py b/pyxb/binding/basis.py index f624e0e6..9b2a7e06 100644 --- a/pyxb/binding/basis.py +++ b/pyxb/binding/basis.py @@ -1023,7 +1023,8 @@ def XsdSuperType (cls): # otherwise directly descends from a Python type; return # the recorded XSD supertype. return cls._XsdBaseType - if issubclass(sc, simpleTypeDefinition): + if (not sc.__name__.startswith('_')) \ + and issubclass(sc, simpleTypeDefinition): return sc raise pyxb.LogicError('No supertype found for %s' % (cls,)) From 028e1f4b16da0bc77730f0afb8f8a4ec0167ebbc Mon Sep 17 00:00:00 2001 From: Oleg Broytman <phd@phdru.name> Date: Tue, 26 Feb 2019 22:29:42 +0300 Subject: [PATCH 09/10] Fix permissions: remove executable bit from non-scripts --- doc/Images/BindingModel.jpg | Bin doc/Images/CTDValidationExceptions.jpg | Bin doc/Images/ComponentModel.jpg | Bin doc/Images/ContentModel.jpg | Bin doc/Images/FACAutomaton.jpg | Bin doc/Images/Namespace.jpg | Bin doc/Images/NamespaceArchive.jpg | Bin doc/Images/NamespaceCore.jpg | Bin doc/Images/RuntimeExceptions.jpg | Bin doc/Images/ScopedDeclarations.jpg | Bin doc/pyxb.eap | Bin editix-pyxb.pre | 0 tests/documents/union/test-union-001.xml | 0 tests/schemas/nested-groups.xsd | 0 tests/schemas/test-union.xsd | 0 15 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 doc/Images/BindingModel.jpg mode change 100755 => 100644 doc/Images/CTDValidationExceptions.jpg mode change 100755 => 100644 doc/Images/ComponentModel.jpg mode change 100755 => 100644 doc/Images/ContentModel.jpg mode change 100755 => 100644 doc/Images/FACAutomaton.jpg mode change 100755 => 100644 doc/Images/Namespace.jpg mode change 100755 => 100644 doc/Images/NamespaceArchive.jpg mode change 100755 => 100644 doc/Images/NamespaceCore.jpg mode change 100755 => 100644 doc/Images/RuntimeExceptions.jpg mode change 100755 => 100644 doc/Images/ScopedDeclarations.jpg mode change 100755 => 100644 doc/pyxb.eap mode change 100755 => 100644 editix-pyxb.pre mode change 100755 => 100644 tests/documents/union/test-union-001.xml mode change 100755 => 100644 tests/schemas/nested-groups.xsd mode change 100755 => 100644 tests/schemas/test-union.xsd diff --git a/doc/Images/BindingModel.jpg b/doc/Images/BindingModel.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/CTDValidationExceptions.jpg b/doc/Images/CTDValidationExceptions.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/ComponentModel.jpg b/doc/Images/ComponentModel.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/ContentModel.jpg b/doc/Images/ContentModel.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/FACAutomaton.jpg b/doc/Images/FACAutomaton.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/Namespace.jpg b/doc/Images/Namespace.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/NamespaceArchive.jpg b/doc/Images/NamespaceArchive.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/NamespaceCore.jpg b/doc/Images/NamespaceCore.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/RuntimeExceptions.jpg b/doc/Images/RuntimeExceptions.jpg old mode 100755 new mode 100644 diff --git a/doc/Images/ScopedDeclarations.jpg b/doc/Images/ScopedDeclarations.jpg old mode 100755 new mode 100644 diff --git a/doc/pyxb.eap b/doc/pyxb.eap old mode 100755 new mode 100644 diff --git a/editix-pyxb.pre b/editix-pyxb.pre old mode 100755 new mode 100644 diff --git a/tests/documents/union/test-union-001.xml b/tests/documents/union/test-union-001.xml old mode 100755 new mode 100644 diff --git a/tests/schemas/nested-groups.xsd b/tests/schemas/nested-groups.xsd old mode 100755 new mode 100644 diff --git a/tests/schemas/test-union.xsd b/tests/schemas/test-union.xsd old mode 100755 new mode 100644 From d74e5ef77e9a5f04930370a03d4d46412fc520c1 Mon Sep 17 00:00:00 2001 From: Oleg Broytman <phd@phdru.name> Date: Tue, 26 Feb 2019 22:53:03 +0300 Subject: [PATCH 10/10] Fix shell scripts: add missing shebang line --- examples/cablelabs/disabled-test.sh | 2 ++ examples/cablelabs/genbindings.sh | 2 ++ examples/dictionary/genbindings.sh | 2 ++ examples/geocoder/disabled-test.sh | 2 ++ examples/geocoder/genbindings.sh | 2 ++ examples/kml/genbindings.sh | 2 ++ examples/manual/demo.sh | 2 ++ examples/manual/demo1.sh | 2 ++ examples/manual/demo2.sh | 2 ++ examples/manual/demo3a.sh | 2 ++ examples/manual/demo3b.sh | 2 ++ examples/manual/demo3c1.sh | 2 ++ examples/manual/demo3c2.sh | 2 ++ examples/manual/demo4.sh | 2 ++ examples/manual/demo4b.sh | 2 ++ examples/ndfd/genbindings.sh | 2 ++ examples/tmsxtvd/disabled-test.sh | 2 ++ examples/tmsxtvd/genbindings.sh | 2 ++ examples/weather/disabled-test.sh | 2 ++ examples/weather/genbindings.sh | 2 ++ examples/xsdprimer/genbindings.sh | 2 ++ maintainer/bundlesupport.sh | 2 ++ maintainer/genpd.sh | 2 ++ maintainer/getw3c.sh | 2 ++ maintainer/usepyxb.sh | 2 ++ tests/complex/nsaugment/test.sh | 2 ++ tests/complex/nsdep/test.sh | 2 ++ tests/complex/nsext/test.sh | 2 ++ tests/drivers/test-stored.sh | 2 ++ tests/support.sh | 2 ++ tests/trac/issue-0048/test.sh | 2 ++ tests/trac/trac-0026/test.sh | 2 ++ tests/trac/trac-0033/test.sh | 2 ++ tests/trac/trac-0043/test.sh | 2 ++ tests/trac/trac-0072/xtest.sh | 2 ++ tests/trac/trac-0073/test.sh | 2 ++ tests/trac/trac-0080/test.sh | 2 ++ tests/trac/trac-0084/test.sh | 2 ++ tests/trac/trac-0088/test.sh | 2 ++ tests/trac/trac-0089/test.sh | 2 ++ tests/trac/trac-0092/test.sh | 2 ++ tests/trac/trac-0108/test.sh | 2 ++ tests/trac/trac-0114/test.sh | 2 ++ tests/trac/trac-0119/test.sh | 2 ++ tests/trac/trac-0133/test.sh | 2 ++ tests/trac/trac-0169/test.sh | 2 ++ tests/trac/trac-0184/test.sh | 2 ++ tests/trac/trac-0186/test.sh | 2 ++ tests/trac/trac-0193/test.sh | 2 ++ tests/trac/trac-0196/test.sh | 2 ++ tests/trac/trac-0197/test.sh | 2 ++ tests/trac/trac-0198/test.sh | 2 ++ tests/trac/trac-0199/test.sh | 2 ++ tests/trac/trac-0202/test.sh | 2 ++ tests/trac/trac-0230/test.sh | 2 ++ 55 files changed, 110 insertions(+) diff --git a/examples/cablelabs/disabled-test.sh b/examples/cablelabs/disabled-test.sh index 5e84c356..07a534b8 100755 --- a/examples/cablelabs/disabled-test.sh +++ b/examples/cablelabs/disabled-test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -f cablelabs.wxs sh genbindings.sh \ && python demo.py diff --git a/examples/cablelabs/genbindings.sh b/examples/cablelabs/genbindings.sh index fb5b21f0..bf0e7fcc 100644 --- a/examples/cablelabs/genbindings.sh +++ b/examples/cablelabs/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + # Note: Module order is in increasing dependency order so schema are # downloaded and saved only once. [ -f cablelabs.wxs ] \ diff --git a/examples/dictionary/genbindings.sh b/examples/dictionary/genbindings.sh index f6eea659..3cba58a0 100644 --- a/examples/dictionary/genbindings.sh +++ b/examples/dictionary/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + URI='http://services.aonaware.com/DictService/DictService.asmx?WSDL' PREFIX='dict' WSDL="${PREFIX}.wsdl" diff --git a/examples/geocoder/disabled-test.sh b/examples/geocoder/disabled-test.sh index 949768a1..b97a61df 100755 --- a/examples/geocoder/disabled-test.sh +++ b/examples/geocoder/disabled-test.sh @@ -1,2 +1,4 @@ +#! /bin/sh + sh genbindings.sh \ && python client.py diff --git a/examples/geocoder/genbindings.sh b/examples/geocoder/genbindings.sh index 1397fe65..55a8a5a0 100644 --- a/examples/geocoder/genbindings.sh +++ b/examples/geocoder/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + GEO_WSDL_URI='http://geocoder.us/dist/eg/clients/GeoCoder.wsdl' GEO_PREFIX=GeoCoder GEO_WSDL="${GEO_PREFIX}.wsdl" diff --git a/examples/kml/genbindings.sh b/examples/kml/genbindings.sh index 8971c549..51e1f7ce 100644 --- a/examples/kml/genbindings.sh +++ b/examples/kml/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYTHONPATH=../.. export PYTHONPATH diff --git a/examples/manual/demo.sh b/examples/manual/demo.sh index c9bd86b4..1065a4cb 100644 --- a/examples/manual/demo.sh +++ b/examples/manual/demo.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -f *.wxs po?.py *.pyc sh demo1.sh \ && python demo1.py > demo1.out \ diff --git a/examples/manual/demo1.sh b/examples/manual/demo1.sh index c1851dc3..e08b1777 100644 --- a/examples/manual/demo1.sh +++ b/examples/manual/demo1.sh @@ -1,2 +1,4 @@ +#! /bin/sh + pyxbgen \ -u po1.xsd -m po1 diff --git a/examples/manual/demo2.sh b/examples/manual/demo2.sh index 26c702ca..39cbaff4 100644 --- a/examples/manual/demo2.sh +++ b/examples/manual/demo2.sh @@ -1,2 +1,4 @@ +#! /bin/sh + pyxbgen \ -u po2.xsd -m po2 diff --git a/examples/manual/demo3a.sh b/examples/manual/demo3a.sh index bf71c4ed..db9a9b85 100644 --- a/examples/manual/demo3a.sh +++ b/examples/manual/demo3a.sh @@ -1,2 +1,4 @@ +#! /bin/sh + pyxbgen \ -u po3.xsd -m po3 diff --git a/examples/manual/demo3b.sh b/examples/manual/demo3b.sh index 34bfd4c6..eccddc42 100644 --- a/examples/manual/demo3b.sh +++ b/examples/manual/demo3b.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u po3.xsd -m po3 \ -u nsaddress.xsd -m address diff --git a/examples/manual/demo3c1.sh b/examples/manual/demo3c1.sh index 8fc69b91..68b6fbe6 100644 --- a/examples/manual/demo3c1.sh +++ b/examples/manual/demo3c1.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u nsaddress.xsd -m address \ --archive-to-file address.wxs diff --git a/examples/manual/demo3c2.sh b/examples/manual/demo3c2.sh index 94b22ed9..2798cc7c 100644 --- a/examples/manual/demo3c2.sh +++ b/examples/manual/demo3c2.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u po3.xsd -m po3 \ --archive-path .:+ diff --git a/examples/manual/demo4.sh b/examples/manual/demo4.sh index 7718e7d7..33a7721f 100644 --- a/examples/manual/demo4.sh +++ b/examples/manual/demo4.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u po4.xsd -m po4 \ --archive-path=.:+ diff --git a/examples/manual/demo4b.sh b/examples/manual/demo4b.sh index 94b22ed9..2798cc7c 100644 --- a/examples/manual/demo4b.sh +++ b/examples/manual/demo4b.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u po3.xsd -m po3 \ --archive-path .:+ diff --git a/examples/ndfd/genbindings.sh b/examples/ndfd/genbindings.sh index 3b6e9886..288edf34 100644 --- a/examples/ndfd/genbindings.sh +++ b/examples/ndfd/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + URI='http://graphical.weather.gov/xml/DWMLgen/schema/DWML.xsd' PREFIX='DWML' diff --git a/examples/tmsxtvd/disabled-test.sh b/examples/tmsxtvd/disabled-test.sh index 84d5e165..b9859018 100755 --- a/examples/tmsxtvd/disabled-test.sh +++ b/examples/tmsxtvd/disabled-test.sh @@ -1,2 +1,4 @@ +#! /bin/sh + sh genbindings.sh python dumpsample.py diff --git a/examples/tmsxtvd/genbindings.sh b/examples/tmsxtvd/genbindings.sh index 6f3a1571..21a7cc7b 100644 --- a/examples/tmsxtvd/genbindings.sh +++ b/examples/tmsxtvd/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + URI='http://docs.tms.tribune.com/tech/xml/schemas/tmsxtvd.xsd' PREFIX='tmstvd' diff --git a/examples/weather/disabled-test.sh b/examples/weather/disabled-test.sh index 8c6e2635..f48285d6 100755 --- a/examples/weather/disabled-test.sh +++ b/examples/weather/disabled-test.sh @@ -1,2 +1,4 @@ +#! /bin/sh + sh genbindings.sh \ && python client_get.py diff --git a/examples/weather/genbindings.sh b/examples/weather/genbindings.sh index 74cd1cad..47e97d3d 100644 --- a/examples/weather/genbindings.sh +++ b/examples/weather/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + WSDL_URI='http://ws.cdyne.com/WeatherWS/Weather.asmx?wsdl' WSDL_URI='http://wsf.cdyne.com/WeatherWS/Weather.asmx?WSDL' wget -O weather.wsdl "${WSDL_URI}" diff --git a/examples/xsdprimer/genbindings.sh b/examples/xsdprimer/genbindings.sh index ad0a220b..1bb945c5 100644 --- a/examples/xsdprimer/genbindings.sh +++ b/examples/xsdprimer/genbindings.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYTHONPATH=../.. export PYTHONPATH rm -rf raw diff --git a/maintainer/bundlesupport.sh b/maintainer/bundlesupport.sh index 5cd3ae29..f72c396b 100644 --- a/maintainer/bundlesupport.sh +++ b/maintainer/bundlesupport.sh @@ -1,3 +1,5 @@ +#! /bin/sh + # This module is sourced by genbind scripts in bundle directories to # initialize a bundle area and provide a function to translate schema. diff --git a/maintainer/genpd.sh b/maintainer/genpd.sh index fa468e52..6b6bd6f5 100644 --- a/maintainer/genpd.sh +++ b/maintainer/genpd.sh @@ -1,3 +1,5 @@ +#! /bin/sh + find pyxb/bundles -name __init__.py \ | sed -e 's@^@"@' -e 's@/[^/]*$@",@' -e 's@/@.@g' \ | fmt diff --git a/maintainer/getw3c.sh b/maintainer/getw3c.sh index f441fa31..948f30d0 100755 --- a/maintainer/getw3c.sh +++ b/maintainer/getw3c.sh @@ -1,3 +1,5 @@ +#! /bin/sh + # Download a mirror of the W3C technical archive # --backup-converted diff --git a/maintainer/usepyxb.sh b/maintainer/usepyxb.sh index 62ff82f4..ae8f4298 100644 --- a/maintainer/usepyxb.sh +++ b/maintainer/usepyxb.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYXB_ROOT=${PYXB_ROOT:-/mnt/devel/pyxb} PYTHONPATH=${PYXB_ROOT}:${PYTHONPATH:+:${PYTHONPATH}} PATH="${PYXB_ROOT}/scripts:${PYXB_ROOT}/bin:${PATH}" diff --git a/tests/complex/nsaugment/test.sh b/tests/complex/nsaugment/test.sh index 695159ca..d5f50721 100755 --- a/tests/complex/nsaugment/test.sh +++ b/tests/complex/nsaugment/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYXB_ARCHIVE_PATH=. export PYXB_ARCHIVE_PATH diff --git a/tests/complex/nsdep/test.sh b/tests/complex/nsdep/test.sh index 3fa80bed..6c6fa92b 100755 --- a/tests/complex/nsdep/test.sh +++ b/tests/complex/nsdep/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYXB_ARCHIVE_PATH=bindings rm -rf bindings mkdir -p bindings diff --git a/tests/complex/nsext/test.sh b/tests/complex/nsext/test.sh index 30508acf..21231b15 100755 --- a/tests/complex/nsext/test.sh +++ b/tests/complex/nsext/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + PYXB_ARCHIVE_PATH=. export PYXB_ARCHIVE_PATH diff --git a/tests/drivers/test-stored.sh b/tests/drivers/test-stored.sh index 88bbe6c2..c12ce05d 100755 --- a/tests/drivers/test-stored.sh +++ b/tests/drivers/test-stored.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -rf bindings mkdir bindings diff --git a/tests/support.sh b/tests/support.sh index 34fc8868..f210f038 100644 --- a/tests/support.sh +++ b/tests/support.sh @@ -1,3 +1,5 @@ +#! /bin/sh + # POSIX shell infrastructure included by various test scripts to # remove redundancy and simplify the scripts. # diff --git a/tests/trac/issue-0048/test.sh b/tests/trac/issue-0048/test.sh index 3b00cb82..5c6e2d6a 100755 --- a/tests/trac/issue-0048/test.sh +++ b/tests/trac/issue-0048/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ --schema-location profile.xsd --module profile python check.py diff --git a/tests/trac/trac-0026/test.sh b/tests/trac/trac-0026/test.sh index 37536e13..0670bb59 100755 --- a/tests/trac/trac-0026/test.sh +++ b/tests/trac/trac-0026/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0033/test.sh b/tests/trac/trac-0033/test.sh index da464a97..023d178c 100755 --- a/tests/trac/trac-0033/test.sh +++ b/tests/trac/trac-0033/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + fail () { echo 1>&2 "${test_name} FAILED: ${@}" exit 1 diff --git a/tests/trac/trac-0043/test.sh b/tests/trac/trac-0043/test.sh index 99cf206f..8be19536 100755 --- a/tests/trac/trac-0043/test.sh +++ b/tests/trac/trac-0043/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ --schema-location=a.xsd --module=A \ --schema-location=b.xsd --module=B \ diff --git a/tests/trac/trac-0072/xtest.sh b/tests/trac/trac-0072/xtest.sh index c1572bad..a0b72881 100755 --- a/tests/trac/trac-0072/xtest.sh +++ b/tests/trac/trac-0072/xtest.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} diff --git a/tests/trac/trac-0073/test.sh b/tests/trac/trac-0073/test.sh index 6b6981d2..d5788ed5 100755 --- a/tests/trac/trac-0073/test.sh +++ b/tests/trac/trac-0073/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ --archive-to-file base.wxs \ --schema-location base.xsd --module base diff --git a/tests/trac/trac-0080/test.sh b/tests/trac/trac-0080/test.sh index b40849bc..f18236c6 100755 --- a/tests/trac/trac-0080/test.sh +++ b/tests/trac/trac-0080/test.sh @@ -1,2 +1,4 @@ +#! /bin/sh + pyxbgen -u multipleRestriction.xsd -m mr python check.py diff --git a/tests/trac/trac-0084/test.sh b/tests/trac/trac-0084/test.sh index 85bab47a..682e3744 100755 --- a/tests/trac/trac-0084/test.sh +++ b/tests/trac/trac-0084/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen -u example.xsd -m example fail () { diff --git a/tests/trac/trac-0088/test.sh b/tests/trac/trac-0088/test.sh index cbbdad78..8b3c09a4 100755 --- a/tests/trac/trac-0088/test.sh +++ b/tests/trac/trac-0088/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + fail () { echo 1>&2 "${test_name} FAILED: ${@}" exit 1 diff --git a/tests/trac/trac-0089/test.sh b/tests/trac/trac-0089/test.sh index 6fdfa868..a984d0e3 100755 --- a/tests/trac/trac-0089/test.sh +++ b/tests/trac/trac-0089/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + fail () { echo 1>&2 "${test_name} FAILED: ${@}" exit 1 diff --git a/tests/trac/trac-0092/test.sh b/tests/trac/trac-0092/test.sh index b03ea8ce..d3cfa4c9 100755 --- a/tests/trac/trac-0092/test.sh +++ b/tests/trac/trac-0092/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -rf bindings mkdir bindings diff --git a/tests/trac/trac-0108/test.sh b/tests/trac/trac-0108/test.sh index df22c93b..95e57074 100755 --- a/tests/trac/trac-0108/test.sh +++ b/tests/trac/trac-0108/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -f TestPatternRestriction.py pyxbgen -u TestPatternRestriction.xsd -m TestPatternRestriction \ && python check.py \ diff --git a/tests/trac/trac-0114/test.sh b/tests/trac/trac-0114/test.sh index fdef3c9f..7cfd4641 100755 --- a/tests/trac/trac-0114/test.sh +++ b/tests/trac/trac-0114/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + rm -f noi.py nois.py *.pyc pyxbgen -u namespace_other_issue.xsd -m noi diff --git a/tests/trac/trac-0119/test.sh b/tests/trac/trac-0119/test.sh index 11bee88b..8073476d 100755 --- a/tests/trac/trac-0119/test.sh +++ b/tests/trac/trac-0119/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + pyxbgen \ -u base.xsd -m base \ -u absent.xsd -m absent diff --git a/tests/trac/trac-0133/test.sh b/tests/trac/trac-0133/test.sh index 2c27472c..bb5287b0 100755 --- a/tests/trac/trac-0133/test.sh +++ b/tests/trac/trac-0133/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0169/test.sh b/tests/trac/trac-0169/test.sh index 44e2acc7..22e22c8d 100755 --- a/tests/trac/trac-0169/test.sh +++ b/tests/trac/trac-0169/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0184/test.sh b/tests/trac/trac-0184/test.sh index 4d2d9cf7..f168c659 100755 --- a/tests/trac/trac-0184/test.sh +++ b/tests/trac/trac-0184/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=trac/184 fail () { diff --git a/tests/trac/trac-0186/test.sh b/tests/trac/trac-0186/test.sh index 3b55ad18..8cdc931a 100755 --- a/tests/trac/trac-0186/test.sh +++ b/tests/trac/trac-0186/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=trac/186 fail () { diff --git a/tests/trac/trac-0193/test.sh b/tests/trac/trac-0193/test.sh index dd099ce6..a3f7d5c5 100755 --- a/tests/trac/trac-0193/test.sh +++ b/tests/trac/trac-0193/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=trac/193 fail () { diff --git a/tests/trac/trac-0196/test.sh b/tests/trac/trac-0196/test.sh index c0c43984..515d839b 100755 --- a/tests/trac/trac-0196/test.sh +++ b/tests/trac/trac-0196/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0197/test.sh b/tests/trac/trac-0197/test.sh index b6da3eba..8b08e09b 100755 --- a/tests/trac/trac-0197/test.sh +++ b/tests/trac/trac-0197/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0198/test.sh b/tests/trac/trac-0198/test.sh index 00a3aac1..aa8cd798 100755 --- a/tests/trac/trac-0198/test.sh +++ b/tests/trac/trac-0198/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0199/test.sh b/tests/trac/trac-0199/test.sh index a335422e..0ac93048 100755 --- a/tests/trac/trac-0199/test.sh +++ b/tests/trac/trac-0199/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0202/test.sh b/tests/trac/trac-0202/test.sh index 5f55f822..46c42b23 100755 --- a/tests/trac/trac-0202/test.sh +++ b/tests/trac/trac-0202/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () { diff --git a/tests/trac/trac-0230/test.sh b/tests/trac/trac-0230/test.sh index c9850bf1..95a1f3c4 100755 --- a/tests/trac/trac-0230/test.sh +++ b/tests/trac/trac-0230/test.sh @@ -1,3 +1,5 @@ +#! /bin/sh + test_name=${0} fail () {
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
