#!/usr/bin/env python
# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
# Released to the public domain.
# Regenerate the Scintilla and SciTE source files that list
# all the lexers and all the properties files.
# Should be run whenever a new lexer is added or removed.
# Requires Python 2.4 or later
# Most files are regenerated in place with templates stored in comments.
# The VS .NET project file is generated into a different file as the
# VS .NET environment will not retain comments when modifying the file.
# The files are copied to a string apart from sections between a
# ++Autogenerated comment and a --Autogenerated comment which is
# generated by the CopyWithInsertion function. After the whole
# string is instantiated, it is compared with the target file and
# if different the file is rewritten.
# Does not regenerate the Visual C++ 6 project files but does the VS .NET
# project file.
import string
import sys
import os
import glob
# EOL constants
CR = "\r"
LF = "\n"
CRLF = "\r\n"
if sys.platform == "win32":
NATIVE = CRLF
else:
# Yes, LF is the native EOL even on Mac OS X. CR is just for
# Mac OS <=9 (a.k.a. "Mac Classic")
NATIVE = LF
# Automatically generated sections contain start and end comments,
# a definition line and the results.
# The results are replaced by regenerating based on the definition line.
# The definition line is a comment prefix followed by "**".
# If there is a digit after the ** then this indicates which list to use
# and the digit and next character are not part of the definition
# Backslash is used as an escape within the definition line.
# The part between \( and \) is repeated for each item in the list.
# \* is replaced by each list item. \t, and \n are tab and newline.
def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists):
copying = 1
listid = 0
output = []
for line in input.splitlines(0):
isStartGenerated = line.startswith(commentPrefix + "++Autogenerated")
if copying and not isStartGenerated:
output.append(line)
if isStartGenerated:
if retainDefs:
output.append(line)
copying = 0
definition = ""
elif not copying and line.startswith(commentPrefix + "**"):
if retainDefs:
output.append(line)
definition = line[len(commentPrefix + "**"):]
if (commentPrefix == "<!--") and (" -->" in definition):
definition = definition.replace(" -->", "")
listid = 0
if definition[0] in string.digits:
listid = int(definition[:1])
definition = definition[2:]
# Hide double slashes as a control character
definition = definition.replace("\\\\", "\001")
# Do some normal C style transforms
definition = definition.replace("\\n", "\n")
definition = definition.replace("\\t", "\t")
# Get the doubled backslashes back as single backslashes
definition = definition.replace("\001", "\\")
startRepeat = definition.find("\\(")
endRepeat = definition.find("\\)")
intro = definition[:startRepeat]
out = ""
if intro.endswith("\n"):
pos = 0
else:
pos = len(intro)
out += intro
middle = definition[startRepeat+2:endRepeat]
for i in lists[listid]:
item = middle.replace("\\*", i)
if pos and (pos + len(item) >= 80):
out += "\\\n"
pos = 0
out += item
pos += len(item)
if item.endswith("\n"):
pos = 0
outro = definition[endRepeat+2:]
out += outro
out = out.replace("\n", eolType) # correct EOLs in generated content
output.append(out)
elif line.startswith(commentPrefix + "--Autogenerated"):
copying = 1
if retainDefs:
output.append(line)
output = [line.rstrip(" \t") for line in output] # trim trailing whitespace
return eolType.join(output) + eolType
def UpdateFile(filename, updated):
""" If the file is different to updated then copy updated
into the file else leave alone so CVS and make don't treat
it as modified. """
try:
infile = open(filename, "rb")
except IOError: # File is not there yet
out = open(filename, "wb")
out.write(updated.encode('utf-8'))
out.close()
print("New %s" % filename)
return
original = infile.read()
infile.close()
original = original.decode('utf-8')
if updated != original:
os.unlink(filename)
out = open(filename, "wb")
out.write(updated.encode('utf-8'))
out.close()
print("Changed %s " % filename)
#~ else:
#~ print "Unchanged", filename
def Generate(inpath, outpath, commentPrefix, eolType, *lists):
"""Generate 'outpath' from 'inpath'.
"eolType" indicates the type of EOLs to use in the generated
file. It should be one of following constants: LF, CRLF,
CR, or NATIVE.
"""
#print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\
# % (inpath, outpath, commentPrefix, eolType)
try:
infile = open(inpath, "rb")
except IOError:
print("Can not open %s" % inpath)
return
original = infile.read()
infile.close()
original = original.decode('utf-8')
updated = CopyWithInsertion(original, commentPrefix,
inpath == outpath, eolType, *lists)
UpdateFile(outpath, updated)
def Regenerate(filename, commentPrefix, eolType, *lists):
"""Regenerate the given file.
"eolType" indicates the type of EOLs to use in the generated
file. It should be one of following constants: LF, CRLF,
CR, or NATIVE.
"""
Generate(filename, filename, commentPrefix, eolType, *lists)
def FindModules(lexFile):
modules = []
f = open(lexFile)
for l in f.readlines():
if l.startswith("LexerModule"):
l = l.replace("(", " ")
modules.append(l.split()[1])
return modules
# Properties that start with lexer. or fold. are automatically found but there are some
# older properties that don't follow this pattern so must be explicitly listed.
knownIrregularProperties = [
"fold",
"styling.within.preprocessor",
"tab.timmy.whinge.level",
"asp.default.language",
"html.tags.case.sensitive",
"ps.level",
"ps.tokenize",
"sql.backslash.escapes",
"nsis.uservars",
"nsis.ignorecase"
]
def FindProperties(lexFile):
properties = {}
f = open(lexFile)
for l in f.readlines():
if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
l = l.strip()
if not l.startswith("//"): # Drop comments
propertyName = l.split("\"")[1]
if propertyName.lower() == propertyName:
# Only allow lower case property names
if propertyName in knownIrregularProperties or \
propertyName.startswith("fold.") or \
propertyName.startswith("lexer."):
properties[propertyName] = 1
return properties
def FindPropertyDocumentation(lexFile):
documents = {}
f = open(lexFile)
name = ""
for l in f.readlines():
l = l.strip()
if "// property " in l:
propertyName = l.split()[2]
if propertyName.lower() == propertyName:
# Only allow lower case property names
name = propertyName
documents[name] = ""
elif "DefineProperty" in l and "\"" in l:
propertyName = l.split("\"")[1]
if propertyName.lower() == propertyName:
# Only allow lower case property names
name = propertyName
documents[name] = ""
elif name:
if l.startswith("//"):
if documents[name]:
documents[name] += " "
documents[name] += l[2:].strip()
elif l.startswith("\""):
l = l[1:].strip()
if l.endswith(";"):
l = l[:-1].strip()
if l.endswith(")"):
l = l[:-1].strip()
if l.endswith("\""):
l = l[:-1]
# Fix escaped double quotes
l = l.replace("\\\"", "\"")
documents[name] += l
else:
name = ""
for name in list(documents.keys()):
if documents[name] == "":
del documents[name]
return documents
def ciCompare(a,b):
return cmp(a.lower(), b.lower())
def ciKey(a):
return a.lower()
def sortListInsensitive(l):
try: # Try key function
l.sort(key=ciKey)
except TypeError: # Earlier version of Python, so use comparison function
l.sort(ciCompare)
def UpdateLineInFile(path, linePrefix, lineReplace):
lines = []
with open(path, "r") as f:
for l in f.readlines():
l = l.rstrip()
if l.startswith(linePrefix):
lines.append(lineReplace)
else:
lines.append(l)
contents = NATIVE.join(lines) + NATIVE
UpdateFile(path, contents)
def UpdateVersionNumbers(root):
with open(root + "scintilla/version.txt") as f:
version = f.read()
versionDotted = version[0] + '.' + version[1] + '.' + version[2]
versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0'
UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA",
"#define VERSION_SCINTILLA \"" + versionDotted + "\"")
UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS",
"#define VERSION_WORDS " + versionCommad)
UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro",
"VERSION =",
"VERSION = " + versionDotted)
UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro",
"VERSION =",
"VERSION = " + versionDotted)
UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", " Release",
" Release " + versionDotted)
UpdateLineInFile(root + "scintilla/doc/index.html",
' <font color="#FFCC99" size="3"> Release version',
' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
if os.path.exists(root + "scite"):
UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE",
"#define VERSION_SCITE \"" + versionDotted + "\"")
UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS",
"#define VERSION_WORDS " + versionCommad)
UpdateLineInFile(root + "scite/doc/SciTEDownload.html", " Release",
" Release " + versionDotted)
UpdateLineInFile(root + "scite/doc/SciTE.html",
' <font color="#FFCC99" size="3"> Release version',
' <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
def RegenerateAll():
root="../../"
# Find all the lexer source code files
lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx")
sortListInsensitive(lexFilePaths)
lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
print(lexFiles)
lexerModules = []
lexerProperties = {}
propertyDocuments = {}
for lexFile in lexFilePaths:
lexerModules.extend(FindModules(lexFile))
for k in FindProperties(lexFile).keys():
lexerProperties[k] = 1
documents = FindPropertyDocumentation(lexFile)
for k in documents.keys():
propertyDocuments[k] = documents[k]
sortListInsensitive(lexerModules)
lexerProperties = list(lexerProperties.keys())
sortListInsensitive(lexerProperties)
# Generate HTML to document each property
# This is done because tags can not be safely put inside comments in HTML
documentProperties = list(propertyDocuments.keys())
sortListInsensitive(documentProperties)
propertiesHTML = []
for k in documentProperties:
propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" %
(k, k, propertyDocuments[k]))
# Find all the SciTE properties files
otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
if os.path.exists(root + "scite"):
propFilePaths = glob.glob(root + "scite/src/*.properties")
sortListInsensitive(propFilePaths)
propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
sortListInsensitive(propFiles)
print(propFiles)
Regenerate(root + "scintilla/src/Catalogue.cxx", "//", NATIVE, lexerModules)
Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles)
Regenerate(root + "scintilla/win32/scintilla_vc6.mak", "#", NATIVE, lexFiles)
if os.path.exists(root + "scite"):
Regenerate(root + "scite/win32/makefile", "#", NATIVE, propFiles)
Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, propFiles)
Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties)
Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", NATIVE, propertiesHTML)
Generate(root + "scite/boundscheck/vcproj.gen",
root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles)
UpdateVersionNumbers(root)
RegenerateAll()
