/*
* Copyright (c) 2007 BUSINESS OBJECTS SOFTWARE LIMITED
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of Business Objects nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* CALCompiler.java
* Created: Feb 23, 2000
* By: Luke Evans
*/
package org.openquark.cal.compiler;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.openquark.cal.compiler.CompilerMessage.AbortCompilation;
import org.openquark.cal.internal.serialization.ModuleSerializationTags;
import org.openquark.cal.internal.serialization.RecordInputStream;
import org.openquark.cal.machine.GeneratedCodeInfo;
import org.openquark.cal.machine.Module;
import org.openquark.cal.module.Cal.Core.CAL_Prelude;
import org.openquark.cal.services.Status;
import org.openquark.cal.util.Graph;
import org.openquark.cal.util.Vertex;
import org.openquark.cal.util.VertexBuilder;
import org.openquark.cal.util.VertexBuilderList;
import org.openquark.util.Pair;
/**
* This is an instance of the CAL compiler.
* Only one compilation job can be active per CALCompiler object.
* Creation date: (2/23/00 5:53:47 AM)
* @author LWE
*/
final class CALCompiler {
/** The namespace for log messages from the compiler package. */
static final private String COMPILER_LOGGER_NAMESPACE = "org.openquark.cal.compiler";
/** An instance of the Java logger used to log compiler messages. */
static final Logger COMPILER_LOGGER = Logger.getLogger(COMPILER_LOGGER_NAMESPACE);
static {
COMPILER_LOGGER.setLevel(Level.FINEST);
}
// Compilation state
/**
* The multiplexed lexer for the CAL grammar.
*
* A multiplexed lexer encapsulates the multiplexing of the two lexers used
* in lexing CAL source: the main CAL lexer and the CALDoc lexer. Since
* these two lexers are codependent (namely on scanning '/''*''*' as the
* start of a CALDoc comment, and '*''/' as the end of one), the two lexers
* should not be created or accessed individually, but rather through this
* field, which guarantees that they are always created and used as a pair.
*/
private CALMultiplexedLexer lexer;
private final CALParser parser;
/** Recognizes valid ASTs. */
private final CALTreeParser treeParser;
private final CALTypeChecker typeChecker;
/**
* An instance of the deprecation scanner which performs a pass for finding deprecated entities.
*/
private final DeprecationScanner deprecationScanner;
/** The Packager */
private Packager packager;
/** Object to log messages generated by compiler or subordinates. Never null. */
private CompilerMessageLogger msgLogger = new MessageLogger();
/** The time stamp for the source of the currently compiling module. */
private long currentModuleTimeStamp;
/**
* used to debug the SourceModel model of CAL's source - how this is done depends on
* the value of DEBUG_SOURCE_MODEL_TEXT
*/
static final private boolean DEBUG_SOURCE_MODEL = false;
/**
* If true, the source model is debugged by converting every successfully parsed module
* into a SourceModel, then converting the SourceModel to module text,
* and trying to parse this again.
* If false, it is debugged by using toParseTree to go directly back to a ParseTreeNode.
*
* Only used if DEBUG_SOURCE_MODEL is true.
*/
static final private boolean DEBUG_SOURCE_MODEL_TEXT = false;
/**
* If true, a SourceModelCopier is used to make a deep copy of the source model,
* and other visitors are created and tested on the source model.
*
* Only used if DEBUG_SOURCE_MODEL is true.
*/
static final private boolean DEBUG_SOURCE_MODEL_VISITOR = false;
/**
* If true, the all source model debugging code paths are enabled.
*/
private static boolean inUnitTestDebugSourceModelMode = false;
/**
* If true, instrumentation of concurrent compilation is enabled.
*/
private static final boolean DEBUG_CONCURRENT_COMPILATION = false;
/**
* Private mutex used for locking instrumentation information shared across all instances of this class.
* Using a zero-length byte[] instead of a new Object() is a small optimization.
*/
private static final byte[] classMutex = new byte[0];
/**
* The number of active threads currently compiling across all instances of this class.
*/
private static int numThreadsCompiling = 0;
/**
* The total number of adjuncts compiled across all instances of this class.
*/
private static int adjunctCompileCount = 0;
/**
* The total number of modules compiled across all instances of this class.
*/
private static int moduleCompileCount = 0;
/**
* A simple container class to hold info about a single import into a module.
* @author Edward Lam
*/
private static class ImportNodeInfo {
private final ModuleName importedName;
private final ParseTreeNode importedModuleNameNode;
/**
* Constructor for a ImportNodeInfo.
* Use this if there no corresponding parse tree node (e.g. defined via source model.).
* @param importedName the name of the imported module.
*/
private ImportNodeInfo(SourceModel.Name.Module importedName) {
if (importedName == null) {
throw new NullPointerException();
}
this.importedName = ModuleName.make(importedName.toSourceText());
this.importedModuleNameNode = null;
}
/**
* Constructor for a ImportNodeInfo.
* Use this if there no corresponding parse tree node (e.g. defined via source model.).
* @param importedName the name of the imported module.
*/
private ImportNodeInfo(ModuleName importedName) {
if (importedName == null) {
throw new NullPointerException();
}
this.importedName = importedName;
this.importedModuleNameNode = null;
}
/**
* Constructor for a ImportNodeInfo from a module import declaration.
* @param importDeclarationNode the parse tree node corresponding to the module import declaration.
*/
private ImportNodeInfo(ParseTreeNode importDeclarationNode) {
if (importDeclarationNode == null) {
throw new NullPointerException();
}
importDeclarationNode.verifyType(CALTreeParserTokenTypes.LITERAL_import);
this.importedModuleNameNode = importDeclarationNode.firstChild();
this.importedName = ModuleNameUtilities.getModuleNameFromParseTree(importedModuleNameNode);
if (this.importedName == null) {
throw new NullPointerException();
}
}
/**
* @return the name of the imported module.
*/
public ModuleName getImportName() {
return importedName;
}
/**
* @return the parse tree node corresponding to the module name in the imported module declaration,
* or null if there was no corresponding parse tree.
*/
public ParseTreeNode getImportedModuleNameNode() {
return importedModuleNameNode;
}
}
/**
* A class to hold a CompiledModuleSourceDefinition, plus info to determine its validity
* gleaned by partial deserialization of its input stream.
*
* Previously, this info was obtained by just partially deserializing the input stream to find the relevant info
* (for instance, the timestamp) wherever it was needed. This caused the input stream to be opened multiple times,
* which is quite slow if the input stream is coming from disk.
*
* @author Edward Lam
*/
private static class CompiledDefinitionInfo {
/** The CompiledModuleSourceDefinition for this info. */
private final CompiledModuleSourceDefinition cmsd;
/** The timestamp from the compiled module source, or -1 if this has not yet been determined. */
private long timeStamp = -1;
/** (Set of ModuleName) The imported modules names from the compiled module source, or null if this has not yet been determined. */
private Set<ModuleName> importedModuleNames;
/** The generated code info from the compiled module source, or null if this has not yet been determined. */
private GeneratedCodeInfo codeInfo;
/**
* Constructor for a CompiledDefinitionInfo.
* @param cmsd
*/
CompiledDefinitionInfo(CompiledModuleSourceDefinition cmsd) {
this.cmsd = cmsd;
}
/**
* Read the header info from the CompiledModuleSourceDefinition if it hasn't already been read.
* @throws IOException
*/
private void initCompiledDefinitionHeaderInfo() throws IOException {
// Check whether initialization has already occurred.
if (importedModuleNames != null) {
return;
}
Status status = new Status("get imports status");
InputStream is = cmsd.getInputStream(status);
if (is == null) {
throw new IOException ("Error reading compiled definition for module " + cmsd.getModuleName() + " : " + status.getMessage());
}
RecordInputStream ris = new RecordInputStream(is);
try {
// read the timestamp.
ris.findRecord(ModuleSerializationTags.SERIALIZATION_INFO);
this.timeStamp = ris.readLong();
// read the code info
this.codeInfo = Module.loadGeneratedCodeInfo(ris);
ris.skipRestOfRecord();
// Read the imports information.
this.importedModuleNames = Module.readDependencies(ris);
} finally {
try {
ris.close();
} catch (IOException e) {
// Ignore this particular exception.
}
}
}
/**
* @return the CompiledModuleSourceDefinition for this info.
*/
public CompiledModuleSourceDefinition getCompiledModuleSourceDefinition() {
return cmsd;
}
/**
* @return The timestamp from the compiled module source.
* @throws IOException if there was a problem reading the compiled module source.
*/
long getTimeStamp() throws IOException {
initCompiledDefinitionHeaderInfo();
return timeStamp;
}
/**
* @return (Set of ModuleName) The imported modules names from the compiled module source.
* @throws IOException if there was a problem reading the compiled module source.
*/
Set<ModuleName> getImportedModuleNames() throws IOException {
initCompiledDefinitionHeaderInfo();
return new HashSet<ModuleName>(importedModuleNames);
}
/**
* @return The generated code info from the compiled module source.
* @throws IOException if there was a problem reading the compiled module source.
*/
GeneratedCodeInfo getCodeInfo() throws IOException {
initCompiledDefinitionHeaderInfo();
return codeInfo;
}
}
/**
* A class to hold info about a number of module source definitions, plus info about whether those definitions are empty.
*
* Previously, this info was obtained wherever it was needed, potentially causing multiple disk accesses if that definition were on disk.
*
* @author Edward Lam
*/
private static class SourceDefinitionsInfo {
/** (ModuleName->ModuleSourceDefinition) Map from module name to source definition. */
private final Map<ModuleName, ModuleSourceDefinition> sourceNameToDefinitionMap;
/** (ModuleSourceDefinition->Boolean) Map from source defn to whether it is empty. */
private final Map<ModuleSourceDefinition, Boolean> sourceDefinitionToEmptyMap = new HashMap<ModuleSourceDefinition, Boolean>();
/**
* Constructor for a SourceDefinitionsInfo.
* @param sourceNameToDefinitionMap
*/
SourceDefinitionsInfo(Map<ModuleName, ModuleSourceDefinition> sourceNameToDefinitionMap) {
this.sourceNameToDefinitionMap = new HashMap<ModuleName, ModuleSourceDefinition>(sourceNameToDefinitionMap);
}
/**
* @param moduleName the name of a module
* @return the corresponding ModuleSourceDefinition, or null if the module is not the name of a module in this info.
*/
public ModuleSourceDefinition getDefinition(ModuleName moduleName) {
return sourceNameToDefinitionMap.get(moduleName);
}
/**
* @return the ModuleSourceDefinitions in this info object.
*/
public Collection<ModuleSourceDefinition> getSourceDefinitions() {
return sourceNameToDefinitionMap.values();
}
/**
* Determines whether the module source is in fact empty.
* If this method is called multiple times, the module source definition will be opened only the first time to determine whether it is empty.
*
* @param msd the module source.
* @return true if the module source is empty.
*/
public boolean isEmptyModuleSource(ModuleSourceDefinition msd) {
/*
* TODOEL: Would it make a significant difference if we instead had this method in ModuleSourceDefinition?
* ie. if there were an abstract method ModuleSourceDefinition.isEmptyModuleSource().
*/
// Check for a value in the map.
Boolean bool = sourceDefinitionToEmptyMap.get(msd);
if (bool == null) {
InputStream is = msd.getInputStream(new Status("Getting module source input stream"));
boolean isEmpty = false;
try {
try {
if (is.read() == -1) {
isEmpty = true;
}
} finally {
is.close();
}
} catch (IOException e) {
}
// Add the value to the map.
bool = Boolean.valueOf(isEmpty);
sourceDefinitionToEmptyMap.put(msd, bool);
}
return bool.booleanValue();
}
}
/**
* This is a simple container class which hold info about the result of parsing out
* module headers (ie. module declaration and imports) from a group of module source definitions.
*
* @author Edward Lam
*/
private static class SourceDefinitionsHeaderParseInfo {
/** The names of modules for which parsing failed. */
private final Set<ModuleName> nonParseableModuleNamesSet;
/** (ModuleName->ImportNodeInfo[]) Map from module name to its import info.
* In order to obtain deterministic ordering of modules in the dependency graph, use a sorted map. */
private final SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap;
/** (ModuleName->ParseTreeNode) Map from module name to its parse tree node, if any.
* For source positions while error checking. */
private final Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap;
SourceDefinitionsHeaderParseInfo(Set<ModuleName> nonParseableModuleNamesSet, SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap, Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap) {
this.nonParseableModuleNamesSet = nonParseableModuleNamesSet;
this.moduleNameToHeaderDefnNodeMap = moduleNameToHeaderDefnNodeMap;
this.moduleNameToImportNodeInfoMap = moduleNameToImportNodeInfoMap;
}
/**
* @return the moduleNameToHeaderDefnNodeMap
*/
public Map<ModuleName, ParseTreeNode> getModuleNameToHeaderDefnNodeMap() {
return moduleNameToHeaderDefnNodeMap;
}
/**
* @return the moduleNameToImportNodeInfoMap
*/
public SortedMap<ModuleName, ImportNodeInfo[]> getModuleNameToImportNodeInfoMap() {
return moduleNameToImportNodeInfoMap;
}
/**
* @return the nonParseableModuleNamesSet
*/
public Set<ModuleName> getNonParseableModuleNamesSet() {
return nonParseableModuleNamesSet;
}
}
/**
* This is the abstract base class for the different kinds of statuses that can be returned
* by the {@link CALCompiler#loadCompiledModule} method.
*
* @author Joseph Wong
*/
private static abstract class CompiledModuleLoadStatus {
/**
* Represents the status where an attempt has been made to load the compiled module after it has been located and
* after it has passed the initial validation checks.
*
* @author Joseph Wong
*/
private static final class LoadAttempted extends CompiledModuleLoadStatus {
/** Constructs an instance of this class. */
private LoadAttempted() {}
}
/**
* Represents the status where the compiled module cannot be found.
*
* @author Joseph Wong
*/
private static final class NotFound extends CompiledModuleLoadStatus {
/** Constructs an instance of this class. */
private NotFound() {}
}
/**
* Represents the status where the compiled module has a timestamp that is older than that of the corresponding module source.
*
* @author Joseph Wong
*/
private static final class OlderThanSource extends CompiledModuleLoadStatus {
/** Constructs an instance of this class. */
private OlderThanSource() {}
}
/**
* Represents the status where one of the modules imported by the compiled module cannot be found.
*
* @author Joseph Wong
*/
private static final class DependeeNotFound extends CompiledModuleLoadStatus {
/**
* The name of the dependee module.
*/
private final ModuleName dependeeName;
/**
* Constructs an instance of this class.
* @param dependeeName the name of the dependee module.
*/
private DependeeNotFound(final ModuleName dependeeName) {
if (dependeeName == null) {
throw new NullPointerException();
}
this.dependeeName = dependeeName;
}
/**
* @return the name of the dependee module.
*/
private ModuleName getDependeeName() {
return dependeeName;
}
}
/**
* Represents the status where the compiled module has a timestamp that is older than that of one of the imported modules.
*
* @author Joseph Wong
*/
private static final class OlderThanDependee extends CompiledModuleLoadStatus {
/**
* The name of the dependee module.
*/
private final ModuleName dependeeName;
/**
* Constructs an instance of this class.
* @param dependeeName the name of the dependee module.
*/
private OlderThanDependee(final ModuleName dependeeName) {
if (dependeeName == null) {
throw new NullPointerException();
}
this.dependeeName = dependeeName;
}
/**
* @return the name of the dependee module.
*/
private ModuleName getDependeeName() {
return dependeeName;
}
}
/**
* Represents the status where an exception is caught or where an internal error has occurred during the processing.
*
* @author Joseph Wong
*/
private static final class ExceptionCaughtOrInternalError extends CompiledModuleLoadStatus {
/** Constructs an instance of this class. */
private ExceptionCaughtOrInternalError() {}
}
/** Private constructor. For use by subclasses only. */
private CompiledModuleLoadStatus() {}
}
/**
* This is the abstract base class for the different kinds of statuses that can be returned
* by timestamp checking methods like {@link CALCompiler#isCompiledSourceUpToDate}.
*
* @author Joseph Wong
*/
private static abstract class TimestampCheckStatus {
/**
* Represents the status where the module of interest is up-to-date.
*
* @author Joseph Wong
*/
private static final class UpToDate extends TimestampCheckStatus {
/** Constructs an instance of this class. */
private UpToDate() {}
}
/**
* Represents the status where the compiled module has a timestamp that is older than that of the corresponding module source.
*
* @author Joseph Wong
*/
private static final class OlderThanSource extends TimestampCheckStatus {
/** Constructs an instance of this class. */
private OlderThanSource() {}
}
/**
* Represents the status where one of the modules imported by the compiled module cannot be found.
*
* @author Joseph Wong
*/
private static final class DependeeNotFound extends TimestampCheckStatus {
/**
* The name of the dependee module.
*/
private final ModuleName dependeeName;
/**
* Constructs an instance of this class.
* @param dependeeName the name of the dependee module.
*/
private DependeeNotFound(final ModuleName dependeeName) {
if (dependeeName == null) {
throw new NullPointerException();
}
this.dependeeName = dependeeName;
}
/**
* @return the name of the dependee module.
*/
private ModuleName getDependeeName() {
return dependeeName;
}
}
/**
* Represents the status where the compiled module has a timestamp that is older than that of one of the imported modules.
*
* @author Joseph Wong
*/
private static final class OlderThanDependee extends TimestampCheckStatus {
/**
* The name of the dependee module.
*/
private final ModuleName dependeeName;
/**
* Constructs an instance of this class.
* @param dependeeName the name of the dependee module.
*/
private OlderThanDependee(final ModuleName dependeeName) {
if (dependeeName == null) {
throw new NullPointerException();
}
this.dependeeName = dependeeName;
}
/**
* @return the name of the dependee module.
*/
private ModuleName getDependeeName() {
return dependeeName;
}
}
/**
* Represents the status where an exception is caught during the processing.
*
* @author Joseph Wong
*/
private static final class ExceptionCaught extends TimestampCheckStatus {
/** Constructs an instance of this class. */
private ExceptionCaught() {}
}
/** Private constructor. For use by subclasses only. */
private TimestampCheckStatus() {}
}
/**
* Construct CALCompiler from a Reader.
*/
CALCompiler() {
// Do the standard initialisations
Reader inStream = new StringReader ("");
// Make a multiplexed lexer
lexer = new CALMultiplexedLexer(this, inStream, null);
// Create a recogniser (parser), it gets its tokens from the multiplexed lexer
parser = new CALParser(this, lexer);
treeParser = new CALTreeParser(this);
// The parser creates AST nodes of type ParseTreeNode.
String treeNodeClassName = ParseTreeNode.class.getName();
parser.setASTNodeClass(treeNodeClassName);
typeChecker = new CALTypeChecker(this);
deprecationScanner = new DeprecationScanner(this);
}
static void setInUnitTestDebugSourceModelMode(boolean debugSourceModel) {
inUnitTestDebugSourceModelMode = debugSourceModel;
}
/**
* Get import info for a module definition defined in a source model.
* @param moduleDefn the module definition.
* @return the import info for the module definition.
*/
private static ImportNodeInfo[] getImportNodeInfo(SourceModel.ModuleDefn moduleDefn) {
ImportNodeInfo[] importedInfoArray = new ImportNodeInfo[moduleDefn.getNImportedModules()];
for (int i = 0; i < importedInfoArray.length; i++) {
importedInfoArray[i] = new ImportNodeInfo(moduleDefn.getNthImportedModule(i).getImportedModuleName());
}
return importedInfoArray;
}
/**
* Get import info for a module definition defined in a parse tree.
* @param moduleDefnNode the module definition.
* @return the import info for the module definition.
*/
private static ImportNodeInfo[] getImportNodeInfo(ParseTreeNode moduleDefnNode) {
moduleDefnNode.verifyType(CALTreeParserTokenTypes.MODULE_DEFN);
ParseTreeNode optionalCALDocNode = moduleDefnNode.firstChild();
optionalCALDocNode.verifyType(CALTreeParserTokenTypes.OPTIONAL_CALDOC_COMMENT);
ParseTreeNode moduleNameNode = optionalCALDocNode.nextSibling();
ParseTreeNode importDeclarationListNode = moduleNameNode.nextSibling();
importDeclarationListNode.verifyType(CALTreeParserTokenTypes.IMPORT_DECLARATION_LIST);
// (List of ImportNodeInfo)
List<ImportNodeInfo> importNodeInfoList = new ArrayList<ImportNodeInfo>();
for (final ParseTreeNode importDeclarationNode : importDeclarationListNode) {
importNodeInfoList.add(new ImportNodeInfo(importDeclarationNode));
}
return importNodeInfoList.toArray(new ImportNodeInfo[importNodeInfoList.size()]);
}
/**
* Compile modules to the current program.
* Modules which are broken and their dependents will not be compiled.
* Note: the modules must not already exist in the program.
* Note: if a module passes parsing but fails type checking, the module will appear in the program, but contain no entities.
* This occurs in the type checker, which calls on the packager to create a new module in the program before type checking.
*
* @param moduleNames - set of names of the modules to be compiled.
* @param sourceDefinitionMap - maps module names to the ModuleSources to compile
* @param compiledDefinitionMap - maps module names to the existing CompiledModuleSources.
* @param packager an object implementing the Packager interface. This object packages and (usually) persists the target form.
* @param foreignContextProvider Any custom foreign context provider specified by the client, or null to use the default.
* Note that in most cases it is sensible to set this as null.
* This is provided as a way for the Eclipse tooling to provide context not visible to the compiler.
* @return CompilerMessage.Severity the highest error condition experienced
*/
CompilerMessage.Severity compileModules(
Set<ModuleName> moduleNames,
Map<ModuleName, ModuleSourceDefinition> sourceDefinitionMap,
Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap,
Packager packager,
ForeignContextProvider foreignContextProvider) {
if (DEBUG_CONCURRENT_COMPILATION) {
// increment the total number of adjuncts compiled, and the number
// of active threads compiling adjuncts
synchronized (classMutex) {
moduleCompileCount++;
System.err.println("module compile count: " + moduleCompileCount);
numThreadsCompiling++;
System.err.println("Num threads: " + numThreadsCompiling);
}
try {
// perform the actual compilation
return compileModulesHelper(moduleNames, sourceDefinitionMap, compiledDefinitionMap, packager, foreignContextProvider);
} finally {
// this thread has finished compiling, so decrement the number of
// active threads compiling adjuncts
synchronized (classMutex) {
numThreadsCompiling--;
}
}
}
else {
return compileModulesHelper(moduleNames, sourceDefinitionMap, compiledDefinitionMap, packager, foreignContextProvider);
}
}
/**
* Determine whether a compiled module source is valid by comparing its timestamp
* to timestamps of the things it depends on.
* @param compiledDefinitionInfo - the compiledDefinitionInfo for the source being checked
* @param msd - ModuleSourceDefintion for the current module.
* @param validCompiledDefinitionInfoMap map from module name to CompiledDefinitionInfo
* @param importedModules - naming the modules imported by the module being checked
* @param sourceDefinitionsInfo info about source definitions to compile
* @return a status reporting whether if the compiled module source is up to date.
*/
private TimestampCheckStatus isCompiledSourceUpToDate (
CompiledDefinitionInfo compiledDefinitionInfo,
ModuleSourceDefinition msd,
Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
Set<ModuleName> importedModules,
SourceDefinitionsInfo sourceDefinitionsInfo) {
CompiledModuleSourceDefinition cmsd = compiledDefinitionInfo.getCompiledModuleSourceDefinition();
// If the module source is more recent than the compiled source
// discard the compiled source.
if (msd.getTimeStamp() > cmsd.getTimeStamp() && !sourceDefinitionsInfo.isEmptyModuleSource(msd)) {
return new TimestampCheckStatus.OlderThanSource();
}
long currentSerializedTimestamp;
try {
currentSerializedTimestamp = compiledDefinitionInfo.getTimeStamp();
} catch (IOException e) {
// Log compiler error message.
ModuleName moduleName = cmsd.getModuleName();
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
return new TimestampCheckStatus.ExceptionCaught();
}
// Now go through the list of imported modules.
// If the compiled source for any imported module is more recent than the compiled
// source for this module than the compiled source for this module is not valid.
for (final ModuleName importName : importedModules) {
CompiledDefinitionInfo importCompiledDefinitionInfo = validCompiledDefinitionInfoMap.get(importName);
// If we can't find a compile module source for the imported module we assume we should re-compile
// this module.
if (importCompiledDefinitionInfo == null) {
return new TimestampCheckStatus.DependeeNotFound(importName);
}
// We want to look at the timestamp actually serialized in the compiled module file.
// This is done because of problems encountered when the timestamps on files are rounded up to
// the nearest second by compression utilities such as winzip.
try {
long importSerializedTimestamp = importCompiledDefinitionInfo.getTimeStamp();
if (importSerializedTimestamp > currentSerializedTimestamp) {
return new TimestampCheckStatus.OlderThanDependee(importName);
}
} catch (IOException e) {
// Log compiler error message.
ModuleName moduleName = cmsd.getModuleName();
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
return new TimestampCheckStatus.ExceptionCaught();
}
}
return new TimestampCheckStatus.UpToDate();
}
/**
* Load a module from a compiled module source.
* Errors will be logged to the compiler's msg logger if the module failed to load.
*
* @param moduleName - name of the module to load
* @param sourceDef - ModuleSourceDefinition for named module (i.e. the CAL source).
* @param validCompiledDefinitionInfoMap - A map of module name -> CompiledDefinitionInfo for modules in the workspace with valid CompiledDefinitionInfo.
* (GeneratedCodeInfo is OK...).
* @param allExistingModules - A map of ModuleName -> Module for all currently existing modules.
* @param foreignClassLoader the classloader to use to resolve foreign classes for the module.
* @param sourceDefinitionsInfo info about source definitions to compile
* @return a status indicating whether an attempt was made to load the module from the compiled module source,
* or if the compiled module source doesn't exist or is out of date.
* @throws IOException
*/
private CompiledModuleLoadStatus loadCompiledModule (
ModuleName moduleName,
ModuleSourceDefinition sourceDef,
Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
Map<ModuleName, Module> allExistingModules,
ClassLoader foreignClassLoader,
SourceDefinitionsInfo sourceDefinitionsInfo) throws IOException {
CompiledDefinitionInfo compiledDefinitionInfo = validCompiledDefinitionInfoMap.get(moduleName);
if (compiledDefinitionInfo == null) {
return new CompiledModuleLoadStatus.NotFound();
}
CompiledModuleSourceDefinition compiledModuleSourceDefinition = compiledDefinitionInfo.getCompiledModuleSourceDefinition();
// Do a quick check of existence and compare time stamp with the CAL source.
// This time stamp comparison is also done by 'isCompiledSourceValid' but
// a quick check here can save opening the input stream and reading the imported
// modules. The CAL source cannot be more recent than the compiled source.
if (compiledModuleSourceDefinition.getTimeStamp() >= sourceDef.getTimeStamp() || sourceDefinitionsInfo.isEmptyModuleSource(sourceDef)) {
int nErrorsBefore = msgLogger.getNErrors();
// The compiled module source has passed the initial quick check of validity.
// Now we want to open it and read its import list. This will allow us to do
// a deeper validity check against the modules it depends on.
Status status = new Status("Compiled definition input stream status.");
try {
// Get the set of imported modules.
// This is needed to check the validity of the current compiled module source.
Set<ModuleName> importedModules = compiledDefinitionInfo.getImportedModuleNames();
final TimestampCheckStatus compiledModuleTimestampCheckStatus =
isCompiledSourceUpToDate (compiledDefinitionInfo, sourceDef, validCompiledDefinitionInfoMap, importedModules, sourceDefinitionsInfo);
// If the compiled source is up-to-date, try to load the module.
// Otherwise, translate the status from isCompiledSourceUpToDate into an appropriate CompiledModuleLoadStatus to be returned.
if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.UpToDate) {
InputStream fis = compiledModuleSourceDefinition.getInputStream(status);
if (fis == null) {
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, " Unable to access backing store.")));
return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();
} else {
// Get a record import stream.
RecordInputStream rs = new RecordInputStream(fis);
try {
Module m = Module.load(rs, allExistingModules, foreignClassLoader, compiledDefinitionInfo.getCodeInfo(), msgLogger);
if (m != null) {
// Succeeded in loading the module.
allExistingModules.put (m.getName(), m);
packager.addModule(m);
}
// Attempted load, continue with next module
return new CompiledModuleLoadStatus.LoadAttempted();
} finally {
rs.close();
}
}
} else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.OlderThanSource) {
return new CompiledModuleLoadStatus.OlderThanSource();
} else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.DependeeNotFound) {
final ModuleName dependeeName = ((TimestampCheckStatus.DependeeNotFound)compiledModuleTimestampCheckStatus).getDependeeName();
return new CompiledModuleLoadStatus.DependeeNotFound(dependeeName);
} else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.OlderThanDependee) {
final ModuleName dependeeName = ((TimestampCheckStatus.OlderThanDependee)compiledModuleTimestampCheckStatus).getDependeeName();
return new CompiledModuleLoadStatus.OlderThanDependee(dependeeName);
} else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.ExceptionCaught) {
return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();
} else {
// unknown kind of TimestampCheckStatus
throw new IllegalStateException("Unknown TimestampCheckStatus: " + compiledModuleTimestampCheckStatus);
}
} catch (Exception e) {
if (msgLogger.getNErrors() > nErrorsBefore || e instanceof UnableToResolveForeignEntityException) {
// If the exception is an UnableToResolveForeignEntityException, there is
// a CompilerMessage inside that we should be logging.
if (e instanceof UnableToResolveForeignEntityException) {
try {
logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
}
//if an error occurred previously, we continue to compile the program to try to report additional
//meaningful compilation errors. However, this can produce spurious exceptions related to the fact
//that the program state does not satisfy preconditions because of the initial error(s). We don't
//report the spurious exception as an internal coding error.
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Info.UnableToRecover()));
} else {
//"Compilation aborted due to an internal error in loading the compiled module {0}. Please contact Business Objects. Detail: {1}"
logMessage(
new CompilerMessage(new SourceRange(moduleName),
new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
}
return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();
}
} else {
return new CompiledModuleLoadStatus.OlderThanSource();
}
}
/**
* Compile modules to the current program.
* Modules which are broken and their dependents will not be compiled.
* Note: the modules must not already exist in the program.
* Note: if a module passes parsing but fails type checking, the module will appear in the program, but contain no entities.
* This occurs in the type checker, which calls on the packager to create a new module in the program before type checking.
*
* @param moduleNames - set of names of the modules to be compiled.
* @param sourceDefinitionMap (ModuleName -> ModuleSourceDefinition) maps module names to the ModuleSources to compile
* @param compiledDefinitionMap (ModuleName -> CompiledModuleSourceDefinition) maps module names to the CompiledModuleSources to load.
* @param packager an object implementing the Packager interface. This object packages and (usually) persists the target form.
* @param foreignContextProvider Any custom foreign context provider specified by the client, or null to use the default.
* @return CompilerMessage.Severity the highest error condition experienced
*/
private CompilerMessage.Severity compileModulesHelper(
Set<ModuleName> moduleNames,
Map<ModuleName, ModuleSourceDefinition> sourceDefinitionMap,
Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap,
Packager packager,
ForeignContextProvider foreignContextProvider) {
if (sourceDefinitionMap == null || packager == null || compiledDefinitionMap == null) {
throw new NullPointerException();
}
// Short circuit if there are no modules to compile.
// This can happen for instance if we try to compile dirty modules only, but find that there are no dirty modules.
if (moduleNames.isEmpty()) {
return CompilerMessage.Severity.INFO;
}
SourceDefinitionsInfo sourceDefinitionsInfo = new SourceDefinitionsInfo(sourceDefinitionMap);
// Store current packager
this.packager = packager;
if (packager == null) {
throw new IllegalArgumentException("CALCompiler.compile: must have a packager.");
}
// Replace the logger.
CompilerMessageLogger oldLogger = msgLogger;
CompilerMessageLogger compileLogger = new MessageLogger(false); // Do not abort compilation with this logger.
msgLogger = compileLogger;
try {
/* the names of modules with errors found during dependency ordering analysis. */
Set<ModuleName> preparseBrokenModuleNameSet = new HashSet<ModuleName>();
/*
* In order to determine dependency ordering, it is necessary to know, for a given source definition,
* the name of the module being represented, plus the modules which it imports.
*
* Previously, this was done by fully parsing the source definitions, and analyzing the parse trees for this info.
* The source model for the module was also generated, since the data for this was readily available.
* However, this led to memory-use issues, as these tree structures are potentially very bulky objects.
* Moreover, they are expensive to construct, meaning that the work done during this initial compilation phase
* could not be done in parallel with other compilation work.
*
* Now, before any real parsing is done, all source definitions are only parsed as far as required in order to determine,
* for a given source definition, the name of the module and its imports. The dependency ordering is calculated
* from this info, and using this ordering, modules are compiled (including parsed) in dependency order.
*/
// Map of ModuleName -> Module which maps module names to modules for all existing/compiled/loaded modules.
Map<ModuleName, Module> allExistingModules = new HashMap<ModuleName, Module>();
//
// Determine dependency ordering.
//
// Gather the names of already packaged modules.
Set<ModuleName> packagedModuleNamesSet = new HashSet<ModuleName>();
for (final Module m : packager.getProgram().getModules()) {
ModuleName moduleName = m.getName();
packagedModuleNamesSet.add(moduleName);
allExistingModules.put (moduleName, m);
}
/*
* Map from module name to the CompiledDefinitionInfo for that module,
* for those definitions which don't need to be regenerated.
*
* This map starts out mapping those modules which contain CompiledDefinitionInfo for those modules for which
* GeneratedCodeInfo.needToRegenerate() returns false.
* In the compile loop below, further pruning occurs -- a module is removed from the map if it is not loaded.
*/
Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap = getValidCodeInfoCompiledDefinitionInfoMap(compiledDefinitionMap);
// The module dependency graph resulting from this phase.
SourceDefinitionsHeaderParseInfo parseInfo =
calculateParseInfo(sourceDefinitionsInfo, validCompiledDefinitionInfoMap, packagedModuleNamesSet, preparseBrokenModuleNameSet);
// Figure out the order in which the modules should be compiled.
Graph<ModuleName> moduleDependencyGraph = getModuleDependencyGraph(parseInfo, packagedModuleNamesSet, preparseBrokenModuleNameSet);
ModuleName[] moduleNamesInCompileOrder = getModuleDependencyOrder(moduleDependencyGraph, parseInfo, preparseBrokenModuleNameSet);
// Calculate the names of broken modules and their dependents (direct or indirect).
Set<ModuleName> dependentBrokenModulesSet = new HashSet<ModuleName>();
for (final ModuleName brokenParseModuleName : preparseBrokenModuleNameSet) {
addDependentModulesToSet(moduleDependencyGraph, brokenParseModuleName, dependentBrokenModulesSet);
}
//
// Compile modules in dependency order.
//
for (final ModuleName moduleName : moduleNamesInCompileOrder) {
// skip if dependent on a broken module
if (dependentBrokenModulesSet.contains(moduleName)) {
continue;
}
// skip if included in graph just because another module depended on it
// (ie: it is not an actual module, but was included for dependency analysis only)
if (packagedModuleNamesSet.contains(moduleName)) {
continue;
}
// replace the compiler's message logger with one to track messages for this module.
CompilerMessageLogger compileModuleLogger = msgLogger;
msgLogger = new MessageLogger(true);
try {
ModuleSourceDefinition sourceDef = sourceDefinitionsInfo.getDefinition(moduleName);
// Get the classloader to use to resolve foreign classes for the module.
// If there is no foreign context provider, use the classloader for this class (a reasonable default).
// For now it is an error if the provided provider does not provide a classloader,
ClassLoader foreignClassLoader = null;
if (foreignContextProvider != null) {
foreignClassLoader = foreignContextProvider.getClassLoader(moduleName);
if (foreignClassLoader == null) {
msgLogger.logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Error.NoClassLoaderProvided(moduleName)));
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
continue;
}
} else {
foreignClassLoader = getClass().getClassLoader();
}
//
// Attempt to load.
//
int nErrorsBeforeLoad = msgLogger.getNErrors();
final CompiledModuleLoadStatus compiledModuleLoadStatus = loadCompiledModule (moduleName, sourceDef, validCompiledDefinitionInfoMap, allExistingModules, foreignClassLoader, sourceDefinitionsInfo);
if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.LoadAttempted) {
// We've loaded the current module from a compiled source.
// Check if there was a problem loading the compiled module.
if (msgLogger.getNErrors() != nErrorsBeforeLoad) {
// Add this module and dependents to the set of broken modules.
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
}
// Continue with the next module.
continue;
}
// If the module source is empty, the module is a sourceless module.
// If we're here, the compiled source has failed to load, so this is a broken module.
if (sourceDefinitionsInfo.isEmptyModuleSource(sourceDef)) {
// We attempt to translate the status returned by loadCompiledModule into a helpful Error message
SourceRange sourceRange = new SourceRange(moduleName);
if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.NotFound) {
msgLogger.logMessage(new CompilerMessage(sourceRange,
new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleName)));
} else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.OlderThanSource) {
msgLogger.logMessage(new CompilerMessage(sourceRange,
new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleName)));
} else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.DependeeNotFound) {
final ModuleName dependeeName = ((CompiledModuleLoadStatus.DependeeNotFound)compiledModuleLoadStatus).getDependeeName();
msgLogger.logMessage(new CompilerMessage(sourceRange,
new MessageKind.Error.CannotLoadSourcelessModuleDependeeNotFound(moduleName, dependeeName)));
} else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.OlderThanDependee) {
final ModuleName dependeeName = ((CompiledModuleLoadStatus.OlderThanDependee)compiledModuleLoadStatus).getDependeeName();
msgLogger.logMessage(new CompilerMessage(sourceRange,
new MessageKind.Error.CannotLoadSourcelessModuleOlderThanDependee(moduleName, dependeeName)));
} else {
// there are other kinds of statuses, e.g. LoadAttempted, ExceptionCaughtOrInternalError
// in these cases we give the generic error message because there aren't more details.
msgLogger.logMessage(new CompilerMessage(sourceRange,
new MessageKind.Error.InvalidSourcelessModule(moduleName)));
}
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
continue;
}
//
// Attempt to compile.
//
boolean moduleCompiledSuccessfully = compileModuleInternal(sourceDef, foreignClassLoader);
if (!moduleCompiledSuccessfully) {
// if compilation fails, add this to the list of broken modules
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
continue;
}
// Add the module to the set of existing modules.
Module currentModule = packager.getCurrentModule();
allExistingModules.put (currentModule.getName(), currentModule);
// If we're here, the module shouldn't be loaded, which means that any previous CompiledDefinitionInfo for it will be out of date
// Note that because we compile modules in dependency order, modules which depend on this one will also be found to need regeneration.
validCompiledDefinitionInfoMap.remove(moduleName);
} catch (CompilerMessage.AbortCompilation e) {
// Compilation aborted
// Make sure that any dependents are added to the set of things that can't be compiled.
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
} catch (Exception e) {
try {
if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {
// If the exception is an UnableToResolveForeignEntityException, there is
// a CompilerMessage inside that we should be logging.
if (e instanceof UnableToResolveForeignEntityException) {
try {
logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
}
//if an error occurred previously while compiling this module, we continue to compile the module to
//try to report additional meaningful compilation errors. However, this can produce spurious exceptions
//related to the fact that the module state does not satisfy preconditions because of the initial error(s).
//We don't report the spurious exception as an internal coding error.
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Info.UnableToRecover()));
// Make sure that any dependents are added to the set of things that can't be compiled.
addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
} else {
// Major failure - internal coding error
logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
}
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
} finally {
// now replace back the compiler's message logger, after adding all the messages from this module.
try {
compileModuleLogger.logMessages(msgLogger);
} finally {
msgLogger = compileModuleLogger;
}
}
}
} catch (Exception e) {
// Catch exceptions which occur during dependency analysis etc.
try {
if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {
// If the exception is an UnableToResolveForeignEntityException, there is
// a CompilerMessage inside that we should be logging.
if (e instanceof UnableToResolveForeignEntityException) {
try {
logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
}
//if an error occurred previously, we continue to try to report additional meaningful compilation errors.
//However, this can produce spurious exceptions related to the fact that the program state does not
//satisfy preconditions because of the initial error(s).
//We don't report the spurious exception as an internal coding error.
logMessage(new CompilerMessage(new MessageKind.Info.UnableToRecover()));
} else {
// Major failure - internal coding error
logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
}
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
} finally {
msgLogger = oldLogger;
try {
msgLogger.logMessages(compileLogger);
} catch (CompilerMessage.AbortCompilation e) {
// The ol' logger has got too many errors.
}
}
//close the packager if there have been no errors otherwise abort
try {
if (compileLogger.getNErrors() == 0) {
packager.close(msgLogger);
} else {
packager.abort(msgLogger);
}
} catch (Packager.PackagerException e) {
try {
ModuleName moduleName = packager.getCurrentModule().getName();
logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Error.UnableToCloseModuleAndPackage(e.toString()) ));
} catch (AbortCompilation ace) {
}
}
return compileLogger.getMaxSeverity();
}
/**
* Adds the dependent modules of the given module to a set.
* @param moduleDependencyGraph the graph.
* @param moduleName the module whose dependents are to be added to the set.
* @param moduleNameSet the set to be modified.
*/
private static void addDependentModulesToSet(Graph<ModuleName> moduleDependencyGraph, ModuleName moduleName, Set<ModuleName> moduleNameSet) {
Set<ModuleName> dependentVertexNames = moduleDependencyGraph.getDependentVertexNames(moduleName);
moduleNameSet.addAll(dependentVertexNames);
}
/**
* Internal method to handle the non-load part of compilation of a single module.
* Compiler messages are logged to the current message logger. This logger should not contain any errors when this method is called.
*
* @param sourceDef the source definition of the module to compile.
* @param foreignClassLoader the classloader to use to resolve foreign classes for the module.
* @return whether the module were compiled without errors.
* @throws Packager.PackagerException thrown if there is a problem wrapping the module
* @throws CompilerMessage.AbortCompilation thrown if there is a problem when logging messages.
*/
private boolean compileModuleInternal(ModuleSourceDefinition sourceDef, ClassLoader foreignClassLoader)
throws Packager.PackagerException, CompilerMessage.AbortCompilation, UnableToResolveForeignEntityException {
ModuleName moduleName = sourceDef.getModuleName();
this.currentModuleTimeStamp = sourceDef.getTimeStamp();
int nErrorsBefore = msgLogger.getNErrors();
//
// Parse the module's source definition.
//
Pair<ParseTreeNode, SourceModel.ModuleDefn> parseResultPair = parseModule(sourceDef);
// Check for parse failure.
if (msgLogger.getNErrors() > nErrorsBefore) {
return false;
}
// The parse tree node.
ParseTreeNode moduleDefnNode = parseResultPair.fst();
// The SourceModel (for metrics processing) - get this before the typechecker can change the parse tree
SourceModel.ModuleDefn moduleDefnSourceModel = parseResultPair.snd();
//
// First, scan the module for @deprecated blocks.
//
getDeprecationScanner().processModule(moduleDefnSourceModel);
//
// Type check the module.
//
typeChecker.checkModule(moduleDefnNode, foreignClassLoader);
// Check for type check failure.
if (msgLogger.getNErrors() > nErrorsBefore) {
return false;
}
//
// Generate the raw and module-level source metric data for this module.
//
ModuleTypeInfo moduleTypeInfo = packager.getModuleTypeInfo(moduleName);
if (moduleDefnSourceModel != null) {
SourceMetricFinder.updateRawMetricData(moduleDefnSourceModel, moduleTypeInfo);
moduleTypeInfo.setModuleSourceMetrics(new ModuleSourceMetrics(moduleTypeInfo));
}
//
// Convert the antlr-generated AST to our own internal machine-specific code.
//
ExpressionGenerator expressionGenerator = new ExpressionGenerator(this, moduleTypeInfo, false);
ParseTreeNode outerDefnListNode = moduleDefnNode.getChild(4);
outerDefnListNode.verifyType(CALTreeParserTokenTypes.OUTER_DEFN_LIST);
expressionGenerator.generateCode(outerDefnListNode);
this.currentModuleTimeStamp = 0L;
//
// Notify the packager that the module is done.
//
packager.wrapModule(msgLogger); // throws PackagerException..
// Return whether the module compiled successfully.
return (msgLogger.getNErrors() == nErrorsBefore);
}
/**
* @param compiledDefinitionMap (ModuleName->CompiledModuleSourceDefinition) map from module name to the compiled module source for that module.
* @return (ModuleName->CompiledDefinitionInfo) Map from module name to the CompiledDefinitionInfo for that module.
* This map will only contain entries for which the GeneratedCodeInfo does not indicate that the code needs to be regenerated.
*/
private Map<ModuleName, CompiledDefinitionInfo> getValidCodeInfoCompiledDefinitionInfoMap(
Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap) {
Map<ModuleName, CompiledDefinitionInfo> result = new HashMap<ModuleName, CompiledDefinitionInfo>();
for (final Map.Entry<ModuleName, CompiledModuleSourceDefinition> entry : compiledDefinitionMap.entrySet()) {
ModuleName moduleName = entry.getKey();
CompiledModuleSourceDefinition cmsd = entry.getValue();
CompiledDefinitionInfo info = new CompiledDefinitionInfo(cmsd);
GeneratedCodeInfo codeInfo;
try {
codeInfo = info.getCodeInfo();
} catch (IOException e) {
// couldn't get the code info.
continue;
}
// Check whether the compiled definition needs to be regenerated.
if (codeInfo.needToRegenerate()) {
continue;
}
result.put(moduleName, info);
}
return result;
}
/**
* Determine the dependency ordering for group of module definitions.
* Avoid generating and keeping parse trees here, as these are potentially very bulky objects.
*
* @param sourceDefinitionsInfo info for the source definitions to compile.
* @param validCompiledDefinitionInfoMap - String -> CompiledDefinitionInfo
* @param packagedModuleNamesSet the names of already packaged modules
* @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
* This collection will be populated by this method.
* @return the parse info for the source definitions.
* Note that parse info will be returned for all source definitions for which this can be determined.
* These source definitions may be broken for other reasons, for instance if the source definition name does
* not match the name parsed out of the definition.
*/
private SourceDefinitionsHeaderParseInfo calculateParseInfo(
SourceDefinitionsInfo sourceDefinitionsInfo,
Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
Set<ModuleName> packagedModuleNamesSet,
Set<ModuleName> preparseBrokenModuleNameSet) {
Set<ModuleName> nonParseableModuleNamesSet = new HashSet<ModuleName>();
/* (ModuleName->ImportNodeInfo[]) Map from module name to its import info. */
// In order to obtain deterministic ordering of modules in the dependency graph, use a sorted map.
SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap = new TreeMap<ModuleName, ImportNodeInfo[]>();
/* (ModuleName->ParseTreeNode) Map from module name to its parse tree node, if any.
* For source positions while error checking. */
Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap = new HashMap<ModuleName, ParseTreeNode>();
// Iterate over the source definitions to get the dependency ordering.
for (final ModuleSourceDefinition sourceDef : sourceDefinitionsInfo.getSourceDefinitions()) {
ModuleName moduleSourceName = sourceDef.getModuleName();
// replace the compiler's message logger with one to track messages for this module.
CompilerMessageLogger compilerLogger = msgLogger;
msgLogger = new MessageLogger();
try {
this.currentModuleTimeStamp = 0L;
// In the case of a source definition that is defined in terms of a SourceModel
// object, we can use the toParseTreeNode method to completely bypass the parser
// and get a parse tree directly. This should be a considerable efficiency gain.
// We also associate the SourceModel itself with the module name, so that we can
// avoid having to generate a redundant SourceModel when it's time to compute metrics
if (sourceDef instanceof SourceModelModuleSource) {
if (moduleNameToImportNodeInfoMap.put(moduleSourceName, getImportNodeInfo(((SourceModelModuleSource)sourceDef).getModuleDefn())) != null) {
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.AttemptedRedefinitionOfModule(moduleSourceName)));
}
} else {
// If the compiled module source definition is valid we can use it.
// We consider the compiled source valid if the CAL source is not
// more recent.
CompiledDefinitionInfo compiledDefinitionInfo = validCompiledDefinitionInfoMap.get(moduleSourceName);
boolean compiledDefinitionUpToDate = false;
boolean loadedImports = false;
if (compiledDefinitionInfo != null) {
CompiledModuleSourceDefinition compiledModuleSourceDefinition = compiledDefinitionInfo.getCompiledModuleSourceDefinition();
if (sourceDef.getTimeStamp() <= compiledModuleSourceDefinition.getTimeStamp()) {
// Read the imports from the compiled module information.
Set<ModuleName> imports = null;
try {
imports = compiledDefinitionInfo.getImportedModuleNames();
} catch (IOException e) {
logMessage(new CompilerMessage(new SourceRange(moduleSourceName),
new MessageKind.Warning.DebugMessage("Failed reading imports from compiled module info for " + moduleSourceName), e));
}
if (imports != null) {
loadedImports = true;
ImportNodeInfo importInfo[] = new ImportNodeInfo [imports.size()];
int index = 0;
for (final ModuleName importedModule : imports) {
importInfo[index++] = new ImportNodeInfo (importedModule);
}
// Check whether this module has already been defined.
if (moduleNameToImportNodeInfoMap.put(compiledModuleSourceDefinition.getModuleName(), importInfo) != null) {
logMessage(new CompilerMessage(new SourceRange(compiledModuleSourceDefinition.getModuleName()),
new MessageKind.Error.AttemptedRedefinitionOfModule(compiledModuleSourceDefinition.getModuleName())));
}
}
compiledDefinitionUpToDate = true;
}
}
if (!loadedImports) {
// Was unable to load imports from compiled module definition parse them out
// of the module source.
Reader moduleReader = sourceDef.getSourceReader(new Status("Read module status."));
if (moduleReader == null) {
logMessage(new CompilerMessage(new SourceRange(moduleSourceName),
new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
continue;
}
boolean isEmptyModuleSource = sourceDefinitionsInfo.isEmptyModuleSource(sourceDef);
if (isEmptyModuleSource) {
SourceRange sourceRange = new SourceRange(moduleSourceName);
// We will try to be friendly and report an appropriate error about the sourceless module if possible
if (compiledDefinitionInfo == null) {
// a sourceless module should have a CMI file, but it's not found - maybe the Car is broken
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleSourceName)));
} else if (!compiledDefinitionUpToDate) {
// the CMI file for a sourceless module should be newer than the empty stub CAL file - maybe the Car is broken
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleSourceName)));
} else {
// the problem is somewhere else, so we report the general error
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.InvalidSourcelessModule(moduleSourceName)));
}
continue;
}
moduleReader = new BufferedReader(moduleReader);
setInputStream(moduleReader, moduleSourceName);
ModuleName moduleName = null;
try {
int nOldErrors = msgLogger.getNErrors();
// Call the parser to parse a module header definition
parser.moduleHeader();
if (msgLogger.getNErrors() == nOldErrors) {
ParseTreeNode moduleHeaderDefnNode = (ParseTreeNode)parser.getAST();
// sanity check
moduleHeaderDefnNode.verifyType(CALTreeParserTokenTypes.MODULE_DEFN);
ParseTreeNode optionalCALDocNode = moduleHeaderDefnNode.firstChild();
optionalCALDocNode.verifyType(CALTreeParserTokenTypes.OPTIONAL_CALDOC_COMMENT);
ParseTreeNode moduleNameNode = optionalCALDocNode.nextSibling();
moduleName = ModuleNameUtilities.getModuleNameFromParseTree(moduleNameNode);
moduleNameToHeaderDefnNodeMap.put(moduleName, moduleHeaderDefnNode);
// check that module name corresponds to source name
if (moduleSourceName != null && !moduleSourceName.equals(moduleName)) {
logMessage(new CompilerMessage(moduleNameNode, new MessageKind.Error.ModuleNameDoesNotCorrespondToSourceName(moduleName, moduleSourceName)));
}
// Check whether this module has already been defined.
if (moduleNameToImportNodeInfoMap.put(moduleName, getImportNodeInfo(moduleHeaderDefnNode)) != null) {
logMessage(new CompilerMessage(moduleNameNode, new MessageKind.Error.AttemptedRedefinitionOfModule(moduleName)));
}
}
} catch (antlr.RecognitionException e) {
// Recognition (syntax) error
final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
} catch (antlr.TokenStreamException e) {
// Bad token stream. Maybe a bad token (eg. a stray "$" sign)
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));
} finally {
// close the reader if any.
if (moduleReader != null) {
try {
moduleReader.close();
} catch (IOException ioe) {
COMPILER_LOGGER.log(Level.FINE, "Problem closing file for module \"" + moduleSourceName + "\"");
}
}
// track whether the parse failed.
if (msgLogger.getMaxSeverity().compareTo (CompilerMessage.Severity.ERROR) >= 0) {
if (moduleName == null) {
// parsing to valid AST failed since the name node was not found;
// so use the name specified by the caller
moduleName = moduleSourceName;
nonParseableModuleNamesSet.add(moduleName);
}
preparseBrokenModuleNameSet.add(moduleName);
}
}
}
}
} finally {
// now replace back the compiler's message logger, after adding all the messages from this module.
try {
compilerLogger.logMessages(msgLogger);
} catch (CompilerMessage.AbortCompilation e) {
// Don't stop if fatal errors are copied over
} finally {
msgLogger = compilerLogger;
}
}
}
return new SourceDefinitionsHeaderParseInfo(nonParseableModuleNamesSet, moduleNameToImportNodeInfoMap, moduleNameToHeaderDefnNodeMap);
}
/**
* Parse a module from its source definition.
* If there are problems parsing the module, errors will be logged to the compiler's message logger.
*
* @param sourceDef the module's source definition.
* @return Pair (ParseTreeNode, SourceModel.ModuleDefn) the resulting parse tree node for the module, and its source model.
* Null if the module source could not be read (in which case a compiler error will be logged).
* If parsing failed, one or both elements of the pair may be null.
*/
private Pair<ParseTreeNode, SourceModel.ModuleDefn> parseModule(ModuleSourceDefinition sourceDef) {
// The name of the module.
ModuleName moduleSourceName = sourceDef.getModuleName();
// The parse tree node.
ParseTreeNode moduleDefnNode = null;
// The SourceModel (for metrics processing) - get this before the typechecker can change the parse tree
SourceModel.ModuleDefn moduleDefnSourceModel = null;
int nOldErrors = msgLogger.getNErrors();
// In the case of a source definition that is defined in terms of a SourceModel
// object, we can use the toParseTreeNode method to completely bypass the parser
// and get a parse tree directly. This should be a considerable efficiency gain.
// We also associate the SourceModel itself with the module name, so that we can
// avoid having to generate a redundant SourceModel when it's time to compute metrics
if (sourceDef instanceof SourceModelModuleSource) {
moduleDefnNode = ((SourceModelModuleSource)sourceDef).getParseTreeNode();
moduleDefnSourceModel = ((SourceModelModuleSource)sourceDef).getModuleDefn();
} else {
Reader moduleReader = sourceDef.getSourceReader(new Status("Read module status."));
if (moduleReader == null) {
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
return null;
}
moduleReader = new BufferedReader(moduleReader);
setInputStream(moduleReader, moduleSourceName);
try {
// Call the parser to parse the module definition
parser.module();
moduleDefnNode = (ParseTreeNode)parser.getAST();
} catch (antlr.RecognitionException e) {
// Recognition (syntax) error
final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
} catch (antlr.TokenStreamException e) {
// Bad token stream. Maybe a bad token (eg. a stray "$" sign)
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));
} finally {
// close the reader if any.
if (moduleReader != null) {
try {
moduleReader.close();
} catch (IOException ioe) {
COMPILER_LOGGER.log(Level.FINE, "Problem closing file for module \"" + moduleSourceName + "\"");
}
}
}
// Only try to create a source model if the parse was successful
if (moduleDefnNode != null && msgLogger.getNErrors() == nOldErrors) {
moduleDefnSourceModel = SourceModelBuilder.buildModuleDefn(moduleDefnNode);
}
}
// If parsing succeeded to produce an AST, continue with this module node..
// Note: only invoke the tree parser if there are no errors when parsing the module.
// Otherwise the tree parser is sure to fail (in a fatal error) and so there is no point doing this.
if (moduleDefnNode != null && msgLogger.getNErrors() == nOldErrors) {
// Debug..
if (DEBUG_SOURCE_MODEL || inUnitTestDebugSourceModelMode) {
//use the generated ParseTree to create a SourceModel.ModuleDefn.
SourceModel.ModuleDefn moduleDefn = SourceModelBuilder.buildModuleDefn(moduleDefnNode);
if (DEBUG_SOURCE_MODEL_VISITOR || inUnitTestDebugSourceModelMode) {
SourceModel.ModuleDefn newModuleDefn = (SourceModel.ModuleDefn)moduleDefn.accept(new SourceModelCopier<Void>(), null);
moduleDefn = newModuleDefn;
moduleDefn.accept(new SourceModelTraverser<Void, Void>(), null);
}
if (DEBUG_SOURCE_MODEL_TEXT || inUnitTestDebugSourceModelMode) {
// ...then create module text from the source model. It should work when parsed again
// i.e. the source model should represent the same CAL as the original CAL text.
ModuleSourceDefinition sourceBuilderModuleSource = new SourceModelModuleSource(moduleDefn);
Reader sourceBuilderModuleReader = sourceBuilderModuleSource.getSourceReader(new Status("Read module status."));
if (sourceBuilderModuleReader == null) {
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
return null;
}
sourceBuilderModuleReader = new BufferedReader(sourceBuilderModuleReader);
setInputStream(sourceBuilderModuleReader, moduleSourceName);
try {
parser.module();
moduleDefnNode = (ParseTreeNode)parser.getAST();
} catch (antlr.RecognitionException e) {
// Recognition (syntax) error
final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
} catch (antlr.TokenStreamException e) {
// Bad token stream. Maybe a bad token (eg. a stray "$" sign)
logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));
}
} else {
moduleDefnNode = moduleDefn.toParseTreeNode();
}
}
// End debug..
//
//Walk the parse tree as a sanity check on the generated AST and of the tree parser
//
try {
treeParser.module(moduleDefnNode);
} catch (antlr.RecognitionException e) {
// Recognition (syntax) error
final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
}
}
return new Pair<ParseTreeNode, SourceModel.ModuleDefn>(moduleDefnNode, moduleDefnSourceModel);
}
/**
* Compile an adjunct from an AdjunctSource.
* @param adjunctSource the adjunct source to compile.
* @param packager an object implementing the Packager interface. This object packages and (usually) persists the target form.
* @param adjunctModuleName the name of the module in which to compile the adjunct.
* @return the highest error condition experienced
*/
CompilerMessage.Severity compileAdjunct(AdjunctSource adjunctSource, Packager packager, ModuleName adjunctModuleName) {
if (DEBUG_CONCURRENT_COMPILATION) {
// increment the total number of adjuncts compiled, and the number
// of active threads compiling adjuncts
synchronized (classMutex) {
adjunctCompileCount++;
System.err.println("adjunct compile count: " + adjunctCompileCount);
numThreadsCompiling++;
System.err.println("Num threads: " + numThreadsCompiling);
}
try {
// perform the actual compilation
return compileAdjunctHelper(adjunctSource, packager, adjunctModuleName);
} finally {
// this thread has finished compiling, so decrement the number of
// active threads compiling adjuncts
synchronized (classMutex) {
numThreadsCompiling--;
}
}
}
else {
return compileAdjunctHelper(adjunctSource, packager, adjunctModuleName);
}
}
/**
* Compile an adjunct from an AdjunctSource.
* @param adjunctSource the adjuct source to compile.
* @param packager an object implementing the Packager interface. This object packages and (usually) persists the target form.
* @param adjunctModuleName the name of the module in which to compile the adjunct.
* @return the highest error condition experienced
*/
private CompilerMessage.Severity compileAdjunctHelper(AdjunctSource adjunctSource, Packager packager, ModuleName adjunctModuleName) {
if (adjunctModuleName == null) {
throw new NullPointerException("CALCompiler.compile: Module name must not be null.");
}
if (packager == null) {
throw new IllegalArgumentException("CALCompiler.compile: must have a packager.");
}
//Store current packager
this.packager = packager;
if (adjunctSource instanceof AdjunctSource.FromText) {
setInputStream(((AdjunctSource.FromText)adjunctSource).getReader(), null);
}
CompilerMessageLogger oldLogger = msgLogger;
CompilerMessageLogger compileLogger = new MessageLogger(true); // Internal compiler logger
setCompilerMessageLogger(compileLogger);
// Compile and catch any fatal errors
//todoBI we need a method of adding scs to a module, and then removing them, without
//typechecking the whole module. In other words, adding adjuncts as an undoable operation.
try {
try {
try {
// Initialize packaging of the adjunct. This will switch to the
// adjunct module and do necessary housekeeping.
packager.initAdjunct(adjunctModuleName);
} catch (Packager.PackagerException e) {
logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Fatal.ModuleNotInWorkspace(adjunctModuleName)));
}
ParseTreeNode parseTree = null;
if (adjunctSource instanceof AdjunctSource.FromText) {
// Call the parser
parser.adjunct();
parseTree = (ParseTreeNode)parser.getAST();
} else if (adjunctSource instanceof AdjunctSource.FromSourceModel) {
parseTree = ((AdjunctSource.FromSourceModel)adjunctSource).toParseTreeNode();
} else {
throw new IllegalArgumentException(
"CALCompiler.compileAdjunct - cannot handle adjunct source of type " + adjunctSource.getClass());
}
//only invoke the tree parser if there are no errors when parsing the module. Otherwise the
//tree parser is sure to fail (in a fatal error) and so there is no point doing this.
if (msgLogger.getNErrors() == 0) {
//Walk the parse tree as a sanity check on the generated AST and of the tree parser
treeParser.adjunct(parseTree);
}
//We must type check the adjunct in order to resolve unqualified references,
//and lift lambdas and local functions defined within the body of the adjunct.
//We type check an outer defn list in order to have a place to attach lifted lambdas.
typeChecker.checkAdjunct(parseTree, adjunctModuleName);
ExpressionGenerator expressionGenerator = new ExpressionGenerator(this, packager.getModuleTypeInfo(adjunctModuleName), true);
expressionGenerator.generateCode(parseTree);
//
// Notify the packager that the adjunct compilation is done.
//
packager.wrapAdjunct(msgLogger); // throws PackagerException..
} catch (antlr.RecognitionException e) {
// Recognition (syntax) error
final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
} catch (antlr.TokenStreamException e) {
// Bad token stream. Maybe a bad token (eg. a stray "$" sign)
logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Error.BadTokenStream(), e));
}
} catch (AbortCompilation e) {
//compilation aborted
} catch (Exception e) {
try {
if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {
// If the exception is an UnableToResolveForeignEntityException, there is
// a CompilerMessage inside that we should be logging.
if (e instanceof UnableToResolveForeignEntityException) {
try {
logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
} catch (AbortCompilation ace) {
//logMessage can throw a AbortCompilation if a FATAL message was sent.
}
}
//if an error occurred previously, we continue to compile the program to try to report additional
//meaningful compilation errors. However, this can produce spurious exceptions related to the fact
//that the program state does not satisfy preconditions because of the initial error(s). We don't
//report the spurious exception as an internal coding error.
logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Info.UnableToRecover()));
} else {
// Major failure - internal coding error
logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
}
} catch (AbortCompilation ace) {
// Yeah, yeah, we know
//raiseError will throw a AbortCompilation since a FATAL message was sent.
}
} finally {
// replace the old logger.
setCompilerMessageLogger(oldLogger);
try {
oldLogger.logMessages(compileLogger);
} catch (AbortCompilation e) {
// too many errors for the ol' logger.
}
}
//close the package if there have been no errors otherwise abort
try {
if (compileLogger.getNErrors() == 0) {
packager.close(msgLogger);
} else {
packager.abort(msgLogger);
}
} catch (org.openquark.cal.compiler.Packager.PackagerException e) {
try {
logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Error.UnableToCloseModuleAndPackage(e.toString())));
} catch (AbortCompilation ace) {
}
}
return compileLogger.getMaxSeverity();
}
/**
* Parse the code text and return the list of identifiers that occur within it.
* This method is used for the initial syntax checking and qualification of
* text within a code gem.
*
* The identifiers returned include free symbols i.e. the symbols in the expression that
* are not defined within the expression itself (e.g. by a let definition). The free symbols can
* be either qualified (Prelude.x, Prelude.Boolean) or unqualified (True, Eq, sin). Those that are
* variables which do not successfully resolve to a top-level symbol must be arguments of the code
* expression.
*
* The list is ordered by increasing source positions of identifiers.
*
* Example:
* " let x :: Boolean; x = and Prelude.True False;
* in and x y ;"
* Returns identifiers : Boolean, and, Prelude.True, False, and, y
*
* Note: Any compiler messages held by this compiler will be lost
*
* @param codeGemBodyText the text of the body of the code gem i.e. what the user actually typed
* @param moduleName the name of the module in which the code gem is considered to exist.
* @param moduleNameResolver the module name resolver for the module in which the code gem is considered to exist.
* @return List (SourceIdentifier) the name, type and position of identifiers encountered in
* the expression. Returns null if the expression does not parse.
*/
List<SourceIdentifier> findIdentifiersInExpression(String codeGemBodyText, ModuleName moduleName, ModuleNameResolver moduleNameResolver) {
if ((codeGemBodyText == null) || (moduleName == null)) {
throw new NullPointerException();
}
java.io.StringReader sourceReader = new java.io.StringReader(codeGemBodyText);
setInputStream(sourceReader, null);
SourceIdentifierFinder<SourceIdentifier, SourceIdentifier> identifierFinder = new SourceIdentifierFinder.AllIdentifierFinder();
return identifierFinder.findIdentifiersInUnparsedExpression(parser, treeParser, msgLogger, moduleName, moduleNameResolver);
}
/**
* Parse the specified module and return all identifier references which need to be affected
* in order to rename the specified top-level identifier without conflicts.
*
* The list of objects returned (RenameData) indicates the changes which must occur in order
* to properly rename the specified symbol (ie: the results will include renamings for references
* to the requested symbol, and may include renamings for local variables if any conflict with the
* new name).
*
* Example:
* Renaming "s" to "r" in the expression
*
* "let r = 1.0;
* r2 = if (s r) then 2.0 else 0.0;
* in
* case x of
* (r, r3) -> and (s r) (r2 < r3);
* ;"
*
* Will return renamings to "r" for references to "s",
* renamings to "r4" for case variable "r" ("r2" is bound in let, "r3" bound in case)
* renamings to "r3" for let variable "r" ("r2" is bound in let)
*
* Notes: The list returned is ordered by increasing source position of identifier names.
* Any compiler messages held by this compiler will be lost
*
* @param source module source to parse
* @param oldName old name of the identifier
* @param newName new name of the identifier
* @param category category of the identifier
* @return List the name and position of renamings encountered in
* the expression. Returns null if the expression does not parse.
*/
List<SourceModification> findRenamingsInModule(ModuleSourceDefinition source, QualifiedName oldName, QualifiedName newName, SourceIdentifier.Category category) {
Reader moduleReader = source.getSourceReader(new Status("Read module status."));
if (moduleReader == null) {
return Collections.emptyList();
}
moduleReader = new BufferedReader(moduleReader);
try {
setInputStream(moduleReader, source.getModuleName());
SourceIdentifierFinder<SourceModification, String> identifierFinder = new RenamedIdentifierFinder(oldName, newName, category);
return identifierFinder.findIdentifiersInUnparsedModule(parser, treeParser);
} finally {
try {
// Close the input stream
moduleReader.close();
} catch (IOException e) {
logMessage(new CompilerMessage(new SourceRange(source.getModuleName()), new MessageKind.Error.UnableToCloseModule(source.getModuleName())));
}
}
}
/**
* Parse the specified expression and return all identifier references which need to be affected in order
* to rename the specified identifier without conflicts.
*
* The list of objects returned (RenameData) indicates the changes which must occur in order
* to properly rename the specified symbol (ie: the results will include renamings for references
* to the requested symbol, and may include renamings for local variables if any conflict with the
* new name).
*
* Example:
* Renaming "s" to "r" in the expression
*
* "let r = 1.0;
* r2 = if (s r) then 2.0 else 0.0;
* in
* case x of
* (r, r3) -> and (s r) (r2 < r3);
* ;"
*
* Will return renamings to "r" for references to "s",
* renamings to "r4" for case variable "r" ("r2" is bound in let, "r3" bound in case)
* renamings to "r3" for let variable "r" ("r2" is bound in let)
*
* @param expressionDefinition the expression to parse
* @param moduleName The name of the module that this expression belongs to
* @param moduleNameResolver the module name resolver for the module to which this expression belongs.
* @param qualificationMap The qualification map for this code expression if it has one, or null if it does not.
* @param oldName old name of the identifier
* @param newName new name of the identifier
* @param category category of the identifier
* @return List the name, type and position of identifiers encountered in the expression,
* ordered by increasing source position of identifier names. Returns null if the expression does not parse.
*/
List<SourceModification> findRenamingsInExpression(String expressionDefinition, ModuleName moduleName, ModuleNameResolver moduleNameResolver, CodeQualificationMap qualificationMap, QualifiedName oldName, QualifiedName newName, SourceIdentifier.Category category) {
Reader expressionReader = new StringReader(expressionDefinition);
if (expressionReader == null) {
return Collections.emptyList();
}
expressionReader = new BufferedReader(expressionReader);
setInputStream(expressionReader, null);
SourceIdentifierFinder<SourceModification, String> identifierFinder = new RenamedIdentifierFinder(oldName, newName, category, qualificationMap);
return identifierFinder.findIdentifiersInUnparsedExpression(parser, treeParser, msgLogger, moduleName, moduleNameResolver);
}
/**
* Return the packager in use.
* Creation date: (3/16/00 9:43:14 PM)
* @return Packager the current packager
*/
Packager getPackager() {
return packager;
}
/**
* Set the packager associated with the compiler.
* @param packager
*/
void setPackager(Packager packager) {
this.packager = packager;
}
/**
* Returns the type checker that was used during compilation. This type checker will have
* information related to the types found in the compilation of the CAL module.
* Creation date: (10/13/00 10:01:31 AM)
* @return CALTypeChecker
*/
CALTypeChecker getTypeChecker() {
return typeChecker;
}
/**
* @return an instance of the deprecation scanner which performs a pass for finding deprecated entities.
*/
DeprecationScanner getDeprecationScanner() {
return deprecationScanner;
}
/**
* Log the given compiler message.
* If the message has severity FATAL, then an exception is thrown to immediately halt
* compilation. Otherwise, if the message has severity ERROR and the maximum number of
* such errors has been reported then a 'too many errors' exception is thrown.
* @param defaultModuleNameForSourceRange the name of the module with respect to which the message will be reported, provided that
* 1) the compiler message does not have a source range set, and 2) the current module is not set.
* If the current module is set, it will be used for the source range
* @param compilerMessage the message to log.
*/
private void logMessage(ModuleName defaultModuleNameForSourceRange, CompilerMessage compilerMessage) {
if (compilerMessage.getSourceRange() == null){
//handle the case of null source ranges by attempting to put a source range that at least identifies
//the module in which the compiler message was logged, even if its precise position cannot be determined.
final CALTypeChecker typeChecker = getTypeChecker();
ModuleName newSourceRangeModuleName = null;
// Attempt to get the current module name from the type checker.
if (typeChecker != null) {
newSourceRangeModuleName = typeChecker.getCurrentModuleName(); // can be null
}
// If unassigned, assign from the module name which was passed in.
if (newSourceRangeModuleName == null) {
newSourceRangeModuleName = defaultModuleNameForSourceRange; // can be null.
}
// If a non-null source range module name was assigned above, use it.
if (newSourceRangeModuleName != null) {
compilerMessage = compilerMessage.copy(new SourceRange(newSourceRangeModuleName));
}
}
msgLogger.logMessage (compilerMessage);
}
/**
* Log the given compiler message.
* If the message has severity FATAL, then an exception is thrown to immediately halt
* compilation. Otherwise, if the message has severity ERROR and the maximum number of
* such errors has been reported then a 'too many errors' exception is thrown.
* @param compilerMessage CompilerMessage the error/warning/info message
* @throws CompilerMessage.AbortCompilation if a FATAL message is received or
* too many ERROR messages are received
*/
void logMessage(CompilerMessage compilerMessage) {
logMessage(null, compilerMessage);
}
/**
* Set a new input stream for the compiler.
* @param inStream java.io.Reader the input stream
* @param compileLocation the location of the source of the code currently being compiled, or null if none.
*/
private void setInputStream(Reader inStream, ModuleName compileLocation) {
// Now set up a new multiplexed lexer and get it hooked up to the compiler
lexer = new CALMultiplexedLexer(this, inStream, null);
// Create a new queue for tokens presented to the parser (from the multiplexed lexer)
parser.setTokenBuffer(new antlr.TokenBuffer(lexer));
if (compileLocation != null) {
setCompileSourceName(compileLocation.toSourceText());
} else {
setCompileSourceName(null);
}
}
/**
* Set the name of the source of the code currently being compiled.
* @param compileSourceName The name of the current compile source, or null if none.
*/
private void setCompileSourceName(String compileSourceName) {
lexer.setFilename(compileSourceName);
parser.setFilename(compileSourceName);
}
/**
* Set the message logger.
* @param logger the logger to set, or null to use a default logger.
*/
void setCompilerMessageLogger (CompilerMessageLogger logger) {
if (logger == null) {
logger = new MessageLogger();
}
this.msgLogger = logger;
}
CompilerMessageLogger getMessageLogger () {
return msgLogger;
}
/**
* Get the module dependency graph for a given set of modules.
*
* @param parseInfo the parse info for the module source definitions.
* @param packagedModuleNamesSet set of module names which already exist in the packager
* @param preparseBrokenModuleNameSet the names of modules with errors during dependency ordering analysis.
* This collection will be populated by this method.
* @return Graph the module dependency graph.
* Note that this graph may have components where the number of modules in the component > 1.
* This indicates a recursive module dependency among those modules.
*/
private Graph<ModuleName> getModuleDependencyGraph(
SourceDefinitionsHeaderParseInfo parseInfo,
Set<ModuleName> packagedModuleNamesSet,
Set<ModuleName> preparseBrokenModuleNameSet) {
// Perform dependency analysis to divide up the modules defined in this compilation unit into
// a topologically ordered set of strongly connected components.
VertexBuilderList<ModuleName> vertexBuilderList =
makeModuleDependencyGraphBuilderList(parseInfo, packagedModuleNamesSet, preparseBrokenModuleNameSet);
// should never fail. It is a redundant check since makeModuleDependencyGraphBuilderList should throw an exception otherwise.
if (!vertexBuilderList.makesValidGraph()) {
throw new IllegalStateException("Internal coding error during dependency analysis.");
}
Graph<ModuleName> g = new Graph<ModuleName>(vertexBuilderList);
g = g.calculateStronglyConnectedComponents();
return g;
}
/**
* Get the order in which the given modules should be compiled such that any dependee modules come before their dependents.
* @param moduleDependencyGraph the module dependency graph.
* @param parseInfo the parse info for the module source definitions.
* Used for source positions while error checking.
* @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
* This collection will be populated by this method.
* @return the module names in the order in which they should be compiled.
*/
private ModuleName[] getModuleDependencyOrder(
Graph<ModuleName> moduleDependencyGraph,
SourceDefinitionsHeaderParseInfo parseInfo,
Set<ModuleName> preparseBrokenModuleNameSet) {
// Construct the module names array.
int nComponents = moduleDependencyGraph.getNStronglyConnectedComponents();
List<ModuleName> moduleNameList = new ArrayList<ModuleName>(nComponents);
for (int i = 0; i < nComponents; ++i) {
Graph<ModuleName>.Component component = moduleDependencyGraph.getStronglyConnectedComponent(i);
if (component.size() != 1) {
StringBuilder cyclicNames = new StringBuilder();
ModuleName cyclicModuleName = null;
for (int j = 0, componentSize = component.size(); j < componentSize; ++j) {
if (j > 0) {
cyclicNames.append(", ");
}
cyclicModuleName = component.getVertex(j).getName();
cyclicNames.append(cyclicModuleName);
moduleNameList.add(cyclicModuleName);
preparseBrokenModuleNameSet.add(cyclicModuleName);
}
ParseTreeNode cyclicModuleNameNode = parseInfo.getModuleNameToHeaderDefnNodeMap().get(cyclicModuleName);
MessageKind errorMessage = new MessageKind.Error.CyclicDependenciesBetweenModules(cyclicNames.toString());
if (cyclicModuleNameNode != null) {
logMessage(new CompilerMessage(cyclicModuleNameNode.getChild(0), errorMessage));
} else {
logMessage(new CompilerMessage(new SourceRange(cyclicModuleName), errorMessage));
}
} else {
Vertex<ModuleName> vertex = component.getVertex(0);
ModuleName currentModuleName = vertex.getName();
moduleNameList.add(currentModuleName);
}
}
return moduleNameList.toArray(new ModuleName[moduleNameList.size()]);
}
/**
* Makes the vertex builder list for modules in this compilation unit.
*
* @param parseInfo the parse info for the module source definitions.
* @param packagedModuleNamesSet set (String) of module names which already exist in the packager
* @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
* This collection will be populated by this method.
* @return VertexBuilderList
*/
private VertexBuilderList<ModuleName> makeModuleDependencyGraphBuilderList(
SourceDefinitionsHeaderParseInfo parseInfo,
Set<ModuleName> packagedModuleNamesSet,
Set<ModuleName> preparseBrokenModuleNameSet) {
Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap = parseInfo.getModuleNameToHeaderDefnNodeMap();
SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap = parseInfo.getModuleNameToImportNodeInfoMap();
Set<ModuleName> nonParseableNamesSet = parseInfo.getNonParseableModuleNamesSet();
VertexBuilderList<ModuleName> vertexBuilderList = new VertexBuilderList<ModuleName>();
// Build up the names of the modules in the set
Set<ModuleName> moduleNamesSet = moduleNameToImportNodeInfoMap.keySet();
// Go through each module and check that the imports make sense. If so, add it to the graph.
for (final Map.Entry<ModuleName, ImportNodeInfo[]> entry : moduleNameToImportNodeInfoMap.entrySet()) {
final ModuleName moduleName = entry.getKey();
// Skip any modules determined to be "broken"
// For example, these may be modules which parse but whose module source names don't correspond to their definitions.
if (preparseBrokenModuleNameSet.contains(moduleName)) {
continue;
}
ImportNodeInfo[] importNodeInfoArray = entry.getValue();
Set<ModuleName> importedModuleNamesSet = new HashSet<ModuleName>();
boolean importsPrelude = false;
boolean broken = false;
for (final ImportNodeInfo importNodeInfo : importNodeInfoArray) {
ModuleName importedModuleName = importNodeInfo.getImportName();
if (moduleName.equals(importedModuleName)) {
SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.AttemptToImportModuleIntoItself(importedModuleName)));
broken = true;
}
if (importedModuleName.equals(CAL_Prelude.MODULE_NAME)) {
importsPrelude = true;
}
if (!moduleNamesSet.contains(importedModuleName) &&
!nonParseableNamesSet.contains(importedModuleName) &&
!packagedModuleNamesSet.contains(importedModuleName)) {
final Set<ModuleName> potentialModules = new HashSet<ModuleName>();
potentialModules.addAll(moduleNamesSet);
potentialModules.addAll(nonParseableNamesSet);
potentialModules.addAll(packagedModuleNamesSet);
final ModuleNameResolver resolverForPotentialModules = ModuleNameResolver.make(potentialModules);
ModuleNameResolver.ResolutionResult resolutionResult = resolverForPotentialModules.resolve(importedModuleName);
SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
if (resolutionResult.isKnownUnambiguous()) {
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithOneSuggestion(importedModuleName, resolutionResult.getResolvedModuleName())));
} else if (resolutionResult.isAmbiguous()) {
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithMultipleSuggestions(importedModuleName, resolutionResult.getPotentialMatches())));
} else {
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithNoSuggestions(importedModuleName)));
}
broken = true;
// do not add this to dependency graph, since it doesn't exist as a vertex
continue;
}
if (!importedModuleNamesSet.add(importedModuleName)) {
SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.RepeatedImportOfModule(importedModuleName)));
broken = true;
}
// Now check whether the module being imported is deprecated
SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
checkResolvedModuleReference(importedModuleName, sourceRange);
}
if (!moduleName.equals(CAL_Prelude.MODULE_NAME) && !importsPrelude) {
//non-Prelude modules must import the Prelude.
ParseTreeNode moduleDefnNode = moduleNameToHeaderDefnNodeMap.get(moduleName);
logMessage(new CompilerMessage(moduleDefnNode, new MessageKind.Error.ModuleMustImportOtherModule(moduleName, CAL_Prelude.MODULE_NAME)));
broken = true;
}
if (broken) {
preparseBrokenModuleNameSet.add(moduleName);
}
vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, importedModuleNamesSet));
}
// Add the names of all packaged and non parseable modules to the graph for completion
for (final ModuleName moduleName : nonParseableNamesSet) {
vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, new HashSet<ModuleName>()));
}
for (final ModuleName moduleName : packagedModuleNamesSet) {
vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, new HashSet<ModuleName>()));
}
return vertexBuilderList;
}
/**
* @param parseTreeNode the parseTreeNode associated with a CompilerMessage. May be null.
* @param defaultModuleNameForSourceRange the module associated with the compiler message.
* @return a SourceRange to use with the CompilerMessage.
* If there is SourceRange available from the provided parseTreeNode (if any), it will be returned.
* Otherwise a default SourceRange will be constructed from the currently compiling module name.
*/
static SourceRange getSourceRangeForCompilerMessage(ParseTreeNode parseTreeNode, ModuleName defaultModuleNameForSourceRange) {
if (parseTreeNode != null) {
SourceRange assemblySourceRange = parseTreeNode.getAssemblySourceRange();
if (assemblySourceRange != null) {
return assemblySourceRange;
}
}
return new SourceRange(defaultModuleNameForSourceRange);
}
/**
* Performs late static checks on a module reference that has already been resolved.
*
* Currently, this performs a deprecation check on a module reference, logging a warning message if the module is deprecated.
* @param moduleName the module name to check.
* @param sourceRange the source range of the reference.
*/
void checkResolvedModuleReference(final ModuleName moduleName, final SourceRange sourceRange) {
if (getDeprecationScanner().isModuleDeprecated(moduleName)) {
logMessage(new CompilerMessage(sourceRange, new MessageKind.Warning.DeprecatedModule(moduleName)));
}
}
/**
* Return the time stamp of the currently compiling module.
* @return long
*/
long getCurrentModuleTimeStamp () {
return currentModuleTimeStamp;
}
/**
* The compiler will use the CAL based optimizer.
*/
void useOptimizer() {
packager.useOptimizer();
}
}