You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
365 lines
12 KiB
365 lines
12 KiB
#
|
|
# This file is part of pysmi software.
|
|
#
|
|
# Copyright (c) 2015-2019, Ilya Etingof <etingof@gmail.com>
|
|
# License: http://snmplabs.com/pysmi/license.html
|
|
#
|
|
import sys
|
|
import re
|
|
import ply.lex as lex
|
|
from pysmi.lexer.base import AbstractLexer
|
|
from pysmi import error
|
|
from pysmi import debug
|
|
|
|
UNSIGNED32_MAX = 4294967295
|
|
UNSIGNED64_MAX = 18446744073709551615
|
|
LEX_VERSION = [int(x) for x in lex.__version__.split('.')]
|
|
|
|
# Do not overload single lexer methods - overload all or none of them!
|
|
# noinspection PySingleQuotedDocstring,PyMethodMayBeStatic,PyIncorrectDocstring
|
|
class SmiV2Lexer(AbstractLexer):
|
|
reserved_words = [
|
|
'ACCESS', 'AGENT-CAPABILITIES', 'APPLICATION', 'AUGMENTS', 'BEGIN', 'BITS',
|
|
'CONTACT-INFO', 'CREATION-REQUIRES', 'Counter', 'Counter32', 'Counter64',
|
|
'DEFINITIONS', 'DEFVAL', 'DESCRIPTION', 'DISPLAY-HINT', 'END', 'ENTERPRISE',
|
|
'EXTENDS', 'FROM', 'GROUP', 'Gauge', 'Gauge32', 'IDENTIFIER', 'IMPLICIT',
|
|
'IMPLIED', 'IMPORTS', 'INCLUDES', 'INDEX', 'INSTALL-ERRORS', 'INTEGER',
|
|
'Integer32', 'IpAddress', 'LAST-UPDATED', 'MANDATORY-GROUPS',
|
|
'MAX-ACCESS', 'MIN-ACCESS', 'MODULE', 'MODULE-COMPLIANCE',
|
|
'MODULE-IDENTITY', 'NOTIFICATION-GROUP', 'NOTIFICATION-TYPE',
|
|
'NOTIFICATIONS', 'OBJECT', 'OBJECT-GROUP', 'OBJECT-IDENTITY', 'OBJECT-TYPE',
|
|
'OBJECTS', 'OCTET', 'OF', 'ORGANIZATION', 'Opaque', 'PIB-ACCESS',
|
|
'PIB-DEFINITIONS', 'PIB-INDEX', 'PIB-MIN-ACCESS', 'PIB-REFERENCES',
|
|
'PIB-TAG', 'POLICY-ACCESS', 'PRODUCT-RELEASE', 'REFERENCE', 'REVISION',
|
|
'SEQUENCE', 'SIZE', 'STATUS', 'STRING', 'SUBJECT-CATEGORIES', 'SUPPORTS',
|
|
'SYNTAX', 'TEXTUAL-CONVENTION', 'TimeTicks', 'TRAP-TYPE', 'UNIQUENESS',
|
|
'UNITS', 'UNIVERSAL', 'Unsigned32', 'VALUE', 'VARIABLES',
|
|
'VARIATION', 'WRITE-SYNTAX'
|
|
]
|
|
|
|
reserved = {}
|
|
for w in reserved_words:
|
|
reserved[w] = w.replace('-', '_').upper()
|
|
# hack to support SMIv1
|
|
if w == 'Counter':
|
|
reserved[w] = 'COUNTER32'
|
|
elif w == 'Gauge':
|
|
reserved[w] = 'GAUGE32'
|
|
|
|
forbidden_words = [
|
|
'ABSENT', 'ANY', 'BIT', 'BOOLEAN', 'BY', 'COMPONENT', 'COMPONENTS',
|
|
'DEFAULT', 'DEFINED', 'ENUMERATED', 'EXPLICIT', 'EXTERNAL', 'FALSE', 'MAX',
|
|
'MIN', 'MINUS-INFINITY', 'NULL', 'OPTIONAL', 'PLUS-INFINITY', 'PRESENT',
|
|
'PRIVATE', 'REAL', 'SET', 'TAGS', 'TRUE', 'WITH'
|
|
]
|
|
|
|
# Token names required!
|
|
tokens = list(set([
|
|
'BIN_STRING',
|
|
'CHOICE',
|
|
'COLON_COLON_EQUAL',
|
|
'DOT_DOT',
|
|
'EXPORTS',
|
|
'HEX_STRING',
|
|
'LOWERCASE_IDENTIFIER',
|
|
'MACRO',
|
|
'NEGATIVENUMBER',
|
|
'NEGATIVENUMBER64',
|
|
'NUMBER',
|
|
'NUMBER64',
|
|
'QUOTED_STRING',
|
|
'UPPERCASE_IDENTIFIER',
|
|
] + list(reserved.values())
|
|
))
|
|
|
|
states = (
|
|
('macro', 'exclusive'),
|
|
('choice', 'exclusive'),
|
|
('exports', 'exclusive'),
|
|
('comment', 'exclusive'),
|
|
)
|
|
|
|
literals = '[]{}():;,-.|'
|
|
|
|
t_DOT_DOT = r'\.\.'
|
|
t_COLON_COLON_EQUAL = r'::='
|
|
|
|
t_ignore = ' \t'
|
|
|
|
def __init__(self, tempdir=''):
|
|
self._tempdir = tempdir
|
|
self.lexer = None
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
if LEX_VERSION < [3, 0]:
|
|
self.lexer = lex.lex(module=self,
|
|
reflags=re.DOTALL,
|
|
outputdir=self._tempdir,
|
|
debug=False)
|
|
else:
|
|
if debug.logger & debug.flagLexer:
|
|
logger = debug.logger.getCurrentLogger()
|
|
else:
|
|
logger = lex.NullLogger()
|
|
|
|
if debug.logger & debug.flagGrammar:
|
|
debuglogger = debug.logger.getCurrentLogger()
|
|
else:
|
|
debuglogger = None
|
|
|
|
self.lexer = lex.lex(module=self,
|
|
reflags=re.DOTALL,
|
|
outputdir=self._tempdir,
|
|
debuglog=debuglogger,
|
|
errorlog=logger)
|
|
|
|
def t_newline(self, t):
|
|
r'\r\n|\n|\r'
|
|
t.lexer.lineno += 1
|
|
|
|
# Skipping MACRO
|
|
def t_MACRO(self, t):
|
|
r'MACRO'
|
|
t.lexer.begin('macro')
|
|
return t
|
|
|
|
def t_macro_newline(self, t):
|
|
r'\r\n|\n|\r'
|
|
t.lexer.lineno += 1
|
|
|
|
def t_macro_END(self, t):
|
|
r'END'
|
|
t.lexer.begin('INITIAL')
|
|
return t
|
|
|
|
def t_macro_body(self, t):
|
|
r'.+?(?=END)'
|
|
pass
|
|
|
|
# Skipping EXPORTS
|
|
def t_EXPORTS(self, t):
|
|
r'EXPORTS'
|
|
t.lexer.begin('exports')
|
|
return t
|
|
|
|
def t_exports_newline(self, t):
|
|
r'\r\n|\n|\r'
|
|
t.lexer.lineno += 1
|
|
|
|
def t_exports_end(self, t):
|
|
r';'
|
|
t.lexer.begin('INITIAL')
|
|
|
|
def t_exports_body(self, t):
|
|
r'[^;]+'
|
|
pass
|
|
|
|
# Skipping CHOICE
|
|
def t_CHOICE(self, t):
|
|
r'CHOICE'
|
|
t.lexer.begin('choice')
|
|
return t
|
|
|
|
def t_choice_newline(self, t):
|
|
r'\r\n|\n|\r'
|
|
t.lexer.lineno += 1
|
|
|
|
def t_choice_end(self, t):
|
|
r'\}'
|
|
t.lexer.begin('INITIAL')
|
|
|
|
def t_choice_body(self, t):
|
|
r'[^\}]+'
|
|
pass
|
|
|
|
# Comment handling
|
|
def t_begin_comment(self, t):
|
|
r'--'
|
|
t.lexer.begin('comment')
|
|
|
|
def t_comment_newline(self, t):
|
|
r'\r\n|\n|\r'
|
|
t.lexer.lineno += 1
|
|
t.lexer.begin('INITIAL')
|
|
|
|
# def t_comment_end(self, t):
|
|
# r'--'
|
|
# t.lexer.begin('INITIAL')
|
|
|
|
def t_comment_body(self, t):
|
|
r'[^\r\n]+'
|
|
pass
|
|
|
|
def t_UPPERCASE_IDENTIFIER(self, t):
|
|
r'[A-Z][-a-zA-z0-9]*'
|
|
if t.value in self.forbidden_words:
|
|
raise error.PySmiLexerError("%s is forbidden" % t.value, lineno=t.lineno)
|
|
|
|
if t.value[-1] == '-':
|
|
raise error.PySmiLexerError("Identifier should not end with '-': %s" % t.value, lineno=t.lineno)
|
|
|
|
t.type = self.reserved.get(t.value, 'UPPERCASE_IDENTIFIER')
|
|
|
|
return t
|
|
|
|
def t_LOWERCASE_IDENTIFIER(self, t):
|
|
r'[0-9]*[a-z][-a-zA-z0-9]*'
|
|
if t.value[-1] == '-':
|
|
raise error.PySmiLexerError("Identifier should not end with '-': %s" % t.value, lineno=t.lineno)
|
|
return t
|
|
|
|
def t_NUMBER(self, t):
|
|
r'-?[0-9]+'
|
|
t.value = int(t.value)
|
|
neg = 0
|
|
if t.value < 0:
|
|
neg = 1
|
|
|
|
val = abs(t.value)
|
|
|
|
if val <= UNSIGNED32_MAX:
|
|
if neg:
|
|
t.type = 'NEGATIVENUMBER'
|
|
|
|
elif val <= UNSIGNED64_MAX:
|
|
if neg:
|
|
t.type = 'NEGATIVENUMBER64'
|
|
else:
|
|
t.type = 'NUMBER64'
|
|
|
|
else:
|
|
raise error.PySmiLexerError("Number %s is too big" % t.value, lineno=t.lineno)
|
|
|
|
return t
|
|
|
|
def t_BIN_STRING(self, t):
|
|
r'\'[01]*\'[bB]'
|
|
value = t.value[1:-2]
|
|
while value and value[0] == '0' and len(value) % 8:
|
|
value = value[1:]
|
|
# XXX raise in strict mode
|
|
# if len(value) % 8:
|
|
# raise error.PySmiLexerError("Number of 0s and 1s have to divide by 8 in binary string %s" % t.value, lineno=t.lineno)
|
|
return t
|
|
|
|
def t_HEX_STRING(self, t):
|
|
r'\'[0-9a-fA-F]*\'[hH]'
|
|
value = t.value[1:-2]
|
|
while value and value[0] == '0' and len(value) % 2:
|
|
value = value[1:]
|
|
# XXX raise in strict mode
|
|
# if len(value) % 2:
|
|
# raise error.PySmiLexerError("Number of symbols have to be even in hex string %s" % t.value, lineno=t.lineno)
|
|
return t
|
|
|
|
def t_QUOTED_STRING(self, t):
|
|
r'\"[^\"]*\"'
|
|
t.lexer.lineno += len(re.findall(r'\r\n|\n|\r', t.value))
|
|
return t
|
|
|
|
def t_error(self, t):
|
|
raise error.PySmiLexerError(
|
|
"Illegal character '%s', %s characters left unparsed at this stage" % (t.value[0], len(t.value) - 1),
|
|
lineno=t.lineno)
|
|
# t.lexer.skip(1)
|
|
|
|
|
|
class SupportSmiV1Keywords(object):
|
|
@staticmethod
|
|
def reserved():
|
|
reserved_words = [
|
|
'ACCESS', 'AGENT-CAPABILITIES', 'APPLICATION', 'AUGMENTS', 'BEGIN', 'BITS',
|
|
'CONTACT-INFO', 'CREATION-REQUIRES', 'Counter', 'Counter32', 'Counter64',
|
|
'DEFINITIONS', 'DEFVAL', 'DESCRIPTION', 'DISPLAY-HINT', 'END', 'ENTERPRISE',
|
|
'EXTENDS', 'FROM', 'GROUP', 'Gauge', 'Gauge32', 'IDENTIFIER', 'IMPLICIT',
|
|
'IMPLIED', 'IMPORTS', 'INCLUDES', 'INDEX', 'INSTALL-ERRORS', 'INTEGER',
|
|
'Integer32', 'IpAddress', 'LAST-UPDATED', 'MANDATORY-GROUPS',
|
|
'MAX-ACCESS', 'MIN-ACCESS', 'MODULE', 'MODULE-COMPLIANCE', 'MAX',
|
|
'MODULE-IDENTITY', 'NetworkAddress', 'NOTIFICATION-GROUP',
|
|
'NOTIFICATION-TYPE', 'NOTIFICATIONS', 'OBJECT', 'OBJECT-GROUP',
|
|
'OBJECT-IDENTITY', 'OBJECT-TYPE', 'OBJECTS', 'OCTET', 'OF', 'ORGANIZATION',
|
|
'Opaque', 'PIB-ACCESS', 'PIB-DEFINITIONS', 'PIB-INDEX', 'PIB-MIN-ACCESS',
|
|
'PIB-REFERENCES', 'PIB-TAG', 'POLICY-ACCESS', 'PRODUCT-RELEASE',
|
|
'REFERENCE', 'REVISION', 'SEQUENCE', 'SIZE', 'STATUS', 'STRING',
|
|
'SUBJECT-CATEGORIES', 'SUPPORTS', 'SYNTAX', 'TEXTUAL-CONVENTION',
|
|
'TimeTicks', 'TRAP-TYPE', 'UNIQUENESS', 'UNITS', 'UNIVERSAL', 'Unsigned32',
|
|
'VALUE', 'VARIABLES', 'VARIATION', 'WRITE-SYNTAX'
|
|
]
|
|
|
|
reserved = {}
|
|
for w in reserved_words:
|
|
reserved[w] = w.replace('-', '_').upper()
|
|
# hack to support SMIv1
|
|
if w == 'Counter':
|
|
reserved[w] = 'COUNTER32'
|
|
elif w == 'Gauge':
|
|
reserved[w] = 'GAUGE32'
|
|
|
|
return reserved
|
|
|
|
@staticmethod
|
|
def forbidden_words():
|
|
return [
|
|
'ABSENT', 'ANY', 'BIT', 'BOOLEAN', 'BY', 'COMPONENT', 'COMPONENTS',
|
|
'DEFAULT', 'DEFINED', 'ENUMERATED', 'EXPLICIT', 'EXTERNAL', 'FALSE',
|
|
'MIN', 'MINUS-INFINITY', 'NULL', 'OPTIONAL', 'PLUS-INFINITY', 'PRESENT',
|
|
'PRIVATE', 'REAL', 'SET', 'TAGS', 'TRUE', 'WITH'
|
|
]
|
|
|
|
@staticmethod
|
|
def tokens():
|
|
# Token names required!
|
|
tokens = [
|
|
'BIN_STRING',
|
|
'CHOICE',
|
|
'COLON_COLON_EQUAL',
|
|
'DOT_DOT',
|
|
'EXPORTS',
|
|
'HEX_STRING',
|
|
'LOWERCASE_IDENTIFIER',
|
|
'MACRO',
|
|
'NEGATIVENUMBER',
|
|
'NEGATIVENUMBER64',
|
|
'NUMBER',
|
|
'NUMBER64',
|
|
'QUOTED_STRING',
|
|
'UPPERCASE_IDENTIFIER',
|
|
]
|
|
tokens += list(SupportSmiV1Keywords.reserved().values())
|
|
return list(set(tokens))
|
|
|
|
|
|
relaxedGrammar = {
|
|
'supportSmiV1Keywords': [
|
|
SupportSmiV1Keywords.reserved,
|
|
SupportSmiV1Keywords.forbidden_words,
|
|
SupportSmiV1Keywords.tokens
|
|
],
|
|
'supportIndex': [],
|
|
'commaAtTheEndOfImport': [],
|
|
'commaAtTheEndOfSequence': [],
|
|
'mixOfCommasAndSpaces': [],
|
|
'uppercaseIdentifier': [],
|
|
'lowcaseIdentifier': [],
|
|
'curlyBracesAroundEnterpriseInTrap': [],
|
|
'noCells': []
|
|
}
|
|
|
|
|
|
def lexerFactory(**grammarOptions):
|
|
classAttr = {}
|
|
|
|
for option in grammarOptions:
|
|
if grammarOptions[option]:
|
|
if option not in relaxedGrammar:
|
|
raise error.PySmiError('Unknown lexer relaxation option: %s' % option)
|
|
|
|
for func in relaxedGrammar[option]:
|
|
if sys.version_info[0] > 2:
|
|
classAttr[func.__name__] = func()
|
|
else:
|
|
classAttr[func.func_name] = func()
|
|
|
|
return type('SmiLexer', (SmiV2Lexer,), classAttr)
|