From 18eade06b4aaf4819e8931f5a2b409cf680839ca Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 9 Apr 2024 13:02:10 +0200 Subject: [PATCH 1/7] Migrated the go language module to the new antlr framework. --- .../java/de/jplag/antlr/AbstractVisitor.java | 22 +- .../jplag/antlr/ContextDelegateVisitor.java | 19 + .../java/de/jplag/antlr/ContextVisitor.java | 96 ++- .../java/de/jplag/antlr/DelegateVisitor.java | 39 ++ .../main/java/de/jplag/antlr/HandlerData.java | 7 +- .../java/de/jplag/antlr/TokenCollector.java | 7 +- languages/golang/pom.xml | 5 + .../main/java/de/jplag/golang/GoLanguage.java | 19 +- .../main/java/de/jplag/golang/GoListener.java | 153 +++++ .../java/de/jplag/golang/GoParserAdapter.java | 69 +- .../java/de/jplag/golang/JPlagGoListener.java | 608 ------------------ 11 files changed, 362 insertions(+), 682 deletions(-) create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java create mode 100644 language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java create mode 100644 languages/golang/src/main/java/de/jplag/golang/GoListener.java delete mode 100644 languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 0ec4c19af..c3ccafa53 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -18,6 +18,7 @@ /** * The abstract visitor. + * * @param The type of the visited entity. */ public abstract class AbstractVisitor { @@ -25,7 +26,7 @@ public abstract class AbstractVisitor { private final Predicate condition; private final List>> entryHandlers; - private TokenType entryTokenType; + protected TokenType entryTokenType; private Function entrySemantics; /** @@ -38,6 +39,7 @@ public abstract class AbstractVisitor { /** * Add an action the visitor runs upon entering the entity. + * * @param handler The action, takes the entity and the variable registry as parameter. * @return Self */ @@ -48,6 +50,7 @@ public AbstractVisitor onEnter(BiConsumer handler) { /** * Add an action the visitor runs upon entering the entity. + * * @param handler The action, takes the entity as parameter. * @return Self */ @@ -58,6 +61,7 @@ public AbstractVisitor onEnter(Consumer handler) { /** * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. + * * @param tokenType The type of the token. * @return Self */ @@ -69,6 +73,7 @@ public AbstractVisitor mapEnter(TokenType tokenType) { /** * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. * Alias for {@link #mapEnter(TokenType)}. + * * @param tokenType The type of the token. * @return Self */ @@ -79,6 +84,7 @@ public AbstractVisitor map(TokenType tokenType) { /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. + * * @param semanticsSupplier A function that takes the entity and returns the semantics. * @return Self */ @@ -89,6 +95,7 @@ public AbstractVisitor withSemantics(Function semanticsSupp /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. + * * @param semanticsSupplier A function that returns the semantics. * @return Self */ @@ -99,6 +106,7 @@ public AbstractVisitor withSemantics(Supplier semanticsSupplie /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type control. + * * @return Self */ public AbstractVisitor withControlSemantics() { @@ -118,15 +126,23 @@ boolean matches(T entity) { * Enter a given entity, injecting the needed dependencies. */ void enter(HandlerData data) { + handleEnter(data, this::extractEnterToken, this::extractEnterToken); + } + + protected void handleEnter(HandlerData data, Function extractStartToken, Function extractEndToken) { if (entryTokenType == null && entrySemantics != null) { logger.warn("Received semantics, but no token type, so no token was generated and the semantics discarded"); } - addToken(data, entryTokenType, entrySemantics, this::extractEnterToken); // addToken takes null token types + addToken(data, entryTokenType, entrySemantics, extractStartToken, extractEndToken); // addToken takes null token types entryHandlers.forEach(handler -> handler.accept(data)); } void addToken(HandlerData data, TokenType tokenType, Function semantics, Function extractToken) { - data.collector().addToken(tokenType, semantics, data.entity(), extractToken, data.variableRegistry()); + addToken(data, tokenType, semantics, extractToken, extractToken); + } + + void addToken(HandlerData data, TokenType tokenType, Function semantics, Function extractStartToken, Function extractEndToken) { + data.collector().addToken(tokenType, semantics, data.entity(), extractStartToken, extractEndToken, data.variableRegistry()); } abstract Token extractEnterToken(T entity); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java new file mode 100644 index 000000000..22dd55f8f --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java @@ -0,0 +1,19 @@ +package de.jplag.antlr; + +import org.antlr.v4.runtime.ParserRuleContext; + +import java.util.function.Function; + +public class ContextDelegateVisitor extends DelegateVisitor { + private ContextVisitor contextVisitor; + + public ContextDelegateVisitor(ContextVisitor delegate, Function mapper) { + super(delegate, mapper); + this.contextVisitor = delegate; + } + + @Override + public void delegateExit(HandlerData parentData) { + this.contextVisitor.exit(parentData.derive(this.mapper)); + } +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 96436248d..dd86ba85d 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -14,23 +14,31 @@ import de.jplag.TokenType; import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableRegistry; +import org.antlr.v4.runtime.tree.TerminalNode; /** * The visitor for nodes, or contexts. + * * @param The antlr type of the node. */ public class ContextVisitor extends AbstractVisitor { private final List>> exitHandlers; private TokenType exitTokenType; private Function exitSemantics; + private boolean lengthAsRange; + + private DelegateVisitor delegate; ContextVisitor(Predicate condition) { super(condition); this.exitHandlers = new ArrayList<>(); + this.lengthAsRange = false; + this.delegate = null; } /** * Add an action the visitor runs upon exiting the entity. + * * @param handler The action, takes the entity and the variable registry as parameter. * @return Self */ @@ -41,6 +49,7 @@ public AbstractVisitor onExit(BiConsumer handler) { /** * Add an action the visitor runs upon exiting the entity. + * * @param handler The action, takes the entity as parameter. * @return Self */ @@ -51,6 +60,7 @@ public AbstractVisitor onExit(Consumer handler) { /** * Tell the visitor that it should generate a token upon exiting the entity. Should only be invoked once per visitor. + * * @param tokenType The type of the token. * @return Self */ @@ -59,11 +69,61 @@ public ContextVisitor mapExit(TokenType tokenType) { return this; } + /** + * Behaves like mapEnter, but the created token will range from the beginning of this context to the end instead of only marking the beginning. + * + * @param tokenType The type of token to crate + * @return Self + */ + public AbstractVisitor mapRange(TokenType tokenType) { + this.entryTokenType = tokenType; + this.lengthAsRange = true; + return this; + } + + /** + * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token. + * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. + * @param mapper The mapper function + */ + public TerminalVisitor delegateTerminal(Function mapper) { + TerminalVisitor delegate = new TerminalVisitor((ignore) -> true); + this.delegate = new DelegateVisitor<>(delegate, parentData -> mapper.apply(parentData).getSymbol()); + return delegate; + } + + /** + * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token. + * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. + * + * Visits the terminal upon exiting this context + * + * @param mapper The mapper function + */ + public TerminalVisitor delegateTerminalExit(Function mapper) { + TerminalVisitor delegate = new TerminalVisitor((ignore) -> true); + this.delegate = new DelegateVisitor<>(delegate, parentData -> mapper.apply(parentData).getSymbol()); + this.delegate.mapOnExit(); + return delegate; + } + + /** + * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token. + * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. + * @param mapper The mapper function + */ + public ContextVisitor delegateContext(Function mapper) { + ContextVisitor visitor = new ContextVisitor<>((ignore) -> true); + this.delegate = new DelegateVisitor<>(visitor, mapper); + return visitor; + } + /** * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be * invoked once per visitor. + * * @param enterTokenType The type of the token generated on enter. - * @param exitTokenType The type of the token generated on exit. + * @param exitTokenType The type of the token generated on exit. * @return Self */ public ContextVisitor mapEnterExit(TokenType enterTokenType, TokenType exitTokenType) { @@ -75,8 +135,9 @@ public ContextVisitor mapEnterExit(TokenType enterTokenType, TokenType exitTo /** * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be * invoked once per visitor. Alias for {@link #mapEnterExit(TokenType, TokenType)}. + * * @param enterTokenType The type of the token generated on enter. - * @param exitTokenType The type of the token generated on exit. + * @param exitTokenType The type of the token generated on exit. * @return Self */ public ContextVisitor map(TokenType enterTokenType, TokenType exitTokenType) { @@ -100,6 +161,7 @@ public ContextVisitor withSemantics(Supplier semantics) { /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type loop begin, * same for the exit and loop end. + * * @return Self */ public ContextVisitor withLoopSemantics() { @@ -110,6 +172,7 @@ public ContextVisitor withLoopSemantics() { /** * Tell the visitor that the entity represents a local scope. + * * @return Self */ public ContextVisitor addLocalScope() { @@ -120,6 +183,7 @@ public ContextVisitor addLocalScope() { /** * Tell the visitor that the entity represents a class scope. + * * @return Self */ public ContextVisitor addClassScope() { @@ -132,12 +196,40 @@ public ContextVisitor addClassScope() { * Exit a given entity, injecting the needed dependencies. */ void exit(HandlerData data) { + if(this.delegate != null) { + this.delegate.delegateExit(data); + return; + } + addToken(data, exitTokenType, exitSemantics, ParserRuleContext::getStop); exitHandlers.forEach(handler -> handler.accept(data)); } + @Override + void enter(HandlerData data) { + if(this.delegate != null) { + this.delegate.delegateEnter(data); + return; + } + + if (this.lengthAsRange) { + this.handleEnter(data, this::extractEnterToken, ParserRuleContext::getStop); + } else { + super.enter(data); + } + } + @Override Token extractEnterToken(T entity) { return entity.getStart(); } + + @Override + boolean matches(T entity) { + if(this.delegate != null && !this.delegate.isPresent(entity)) { + return false; + } + + return super.matches(entity); + } } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java new file mode 100644 index 000000000..bea70d851 --- /dev/null +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java @@ -0,0 +1,39 @@ +package de.jplag.antlr; + +import java.util.function.Function; + +public class DelegateVisitor { + private final AbstractVisitor delegate; + protected final Function mapper; + private boolean mapOnExit; + + public DelegateVisitor(AbstractVisitor delegate, Function mapper) { + this.delegate = delegate; + this.mapper = mapper; + this.mapOnExit = false; + } + + public void delegateEnter(HandlerData parentData) { + if (!this.mapOnExit) { + this.delegate.enter(parentData.derive(this.mapper)); + } + } + + public void mapOnExit() { + this.mapOnExit = true; + } + + public void delegateExit(HandlerData parentData) { + if (this.mapOnExit) { + this.delegate.enter(parentData.derive(this.mapper)); + } + } + + public boolean isPresent(T entity) { + try { + return this.mapper.apply(entity) != null; + } catch (Exception e) { //If something goes wrong during mapping, the delegate is not present + return false; + } + } +} diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java index 6c1bb4095..7e96e8cdd 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java @@ -2,8 +2,13 @@ import de.jplag.semantics.VariableRegistry; +import java.util.function.Function; + /** * Holds the data passed to the (quasi-static) listeners. */ -record HandlerData(T entity, VariableRegistry variableRegistry, TokenCollector collector) { +public record HandlerData(T entity, VariableRegistry variableRegistry, TokenCollector collector) { + public HandlerData derive(Function mapper) { + return new HandlerData<>(mapper.apply(entity), variableRegistry, collector); + } } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index 150088a92..588c471b2 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -40,14 +40,15 @@ List getTokens() { } void addToken(TokenType jplagType, Function semanticsSupplier, T entity, - Function extractToken, VariableRegistry variableRegistry) { + Function extractStartToken, Function extractEndToken, VariableRegistry variableRegistry) { if (jplagType == null) { return; } - org.antlr.v4.runtime.Token antlrToken = extractToken.apply(entity); + org.antlr.v4.runtime.Token antlrToken = extractStartToken.apply(entity); + org.antlr.v4.runtime.Token antlrEndToken = extractEndToken.apply(entity); int line = antlrToken.getLine(); int column = antlrToken.getCharPositionInLine() + 1; - int length = antlrToken.getText().length(); + int length = (antlrEndToken.getStartIndex() - antlrToken.getStartIndex()) + antlrEndToken.getText().length(); Token token; if (extractsSemantics) { if (semanticsSupplier == null) { diff --git a/languages/golang/pom.xml b/languages/golang/pom.xml index a3f044fc3..ca240d77c 100644 --- a/languages/golang/pom.xml +++ b/languages/golang/pom.xml @@ -14,6 +14,11 @@ org.antlr antlr4-runtime + + de.jplag + language-antlr-utils + ${revision} + diff --git a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java index 3dbd09ec4..440bd2d37 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java @@ -1,25 +1,17 @@ package de.jplag.golang; -import java.io.File; -import java.util.List; -import java.util.Set; - +import de.jplag.antlr.AbstractAntlrLanguage; import org.kohsuke.MetaInfServices; -import de.jplag.ParsingException; -import de.jplag.Token; - @MetaInfServices(de.jplag.Language.class) -public class GoLanguage implements de.jplag.Language { - +public class GoLanguage extends AbstractAntlrLanguage { private static final String NAME = "Go Parser"; private static final String IDENTIFIER = "go"; private static final int DEFAULT_MIN_TOKEN_MATCH = 8; private static final String[] FILE_EXTENSIONS = {".go"}; - private final GoParserAdapter parserAdapter; public GoLanguage() { - this.parserAdapter = new GoParserAdapter(); + super(new GoParserAdapter()); } @Override @@ -41,9 +33,4 @@ public String getIdentifier() { public int minimumTokenMatch() { return DEFAULT_MIN_TOKEN_MATCH; } - - @Override - public List parse(Set files, boolean normalize) throws ParsingException { - return parserAdapter.parse(files); - } } diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java new file mode 100644 index 000000000..a67125e6c --- /dev/null +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -0,0 +1,153 @@ +package de.jplag.golang; + +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.golang.grammar.GoParser.*; + +import java.util.Objects; + +import static de.jplag.golang.GoTokenType.*; + +public class GoListener extends AbstractAntlrListener { + public GoListener() { + metaDeclarations(); + + interfaceDeclarations(); + structDeclarations(); + + functionDeclarations(); + + controlFlowRules(); + statements(); + + objectCreation(); + controlFlowKeywords(); + } + + private void metaDeclarations() { + visit(PackageClauseContext.class).mapRange(PACKAGE); + + visit(ImportDeclContext.class).map(IMPORT_DECLARATION); + visit(ImportDeclContext.class).delegateTerminal(ImportDeclContext::L_PAREN).map(IMPORT_CLAUSE_BEGIN); + visit(ImportDeclContext.class).delegateTerminalExit(ImportDeclContext::R_PAREN).map(IMPORT_CLAUSE_END); + + visit(ImportSpecContext.class).mapRange(IMPORT_CLAUSE); + } + + private void interfaceDeclarations() { + visit(InterfaceTypeContext.class).mapEnter(INTERFACE_DECLARATION); + visit(InterfaceTypeContext.class).delegateTerminal(InterfaceTypeContext::L_CURLY).map(INTERFACE_BLOCK_BEGIN); + visit(InterfaceTypeContext.class).delegateTerminalExit(InterfaceTypeContext::R_CURLY).map(INTERFACE_BLOCK_END); + } + + private void structDeclarations() { + visit(StructTypeContext.class).map(STRUCT_DECLARATION); + visit(StructTypeContext.class).delegateTerminal(StructTypeContext::L_CURLY).map(STRUCT_BODY_BEGIN); + visit(StructTypeContext.class).delegateTerminalExit(StructTypeContext::R_CURLY).map(STRUCT_BODY_END); + + visit(FieldDeclContext.class).mapRange(MEMBER_DECLARATION); + } + + private void functionDeclarations() { + visit(FunctionDeclContext.class).map(FUNCTION_DECLARATION); + visit(FunctionDeclContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FUNCTION_BODY_BEGIN); + visit(FunctionDeclContext.class).delegateTerminalExit(context -> context.block().R_CURLY()).map(FUNCTION_BODY_END); + + visit(MethodDeclContext.class).map(FUNCTION_DECLARATION); + visit(MethodDeclContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FUNCTION_BODY_BEGIN); + visit(MethodDeclContext.class).delegateTerminalExit(context -> context.block().R_CURLY()).map(FUNCTION_BODY_END); + + visit(ParameterDeclContext.class, context -> !(context.parent.parent instanceof ReceiverContext)).mapRange(FUNCTION_PARAMETER); + visit(ParameterDeclContext.class, context -> (context.parent.parent instanceof ReceiverContext)).mapRange(RECEIVER); + } + + private void controlFlowRules() { + visit(IfStmtContext.class).map(IF_STATEMENT); + visit(IfStmtContext.class).delegateTerminal(context -> context.block(0).L_CURLY()).map(IF_BLOCK_BEGIN); + visit(IfStmtContext.class).delegateTerminal(context -> context.block(0).R_CURLY()).map(IF_BLOCK_END); + //TODO no else token? + visit(IfStmtContext.class, context -> context.ELSE() != null).delegateTerminal(context -> context.block(1).L_CURLY()).map(ELSE_BLOCK_BEGIN); + visit(IfStmtContext.class, context -> context.ELSE() != null).delegateTerminal(context -> context.block(1).L_CURLY()).map(ELSE_BLOCK_END); + + visit(ForStmtContext.class).map(FOR_STATEMENT); + visit(ForStmtContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FOR_BLOCK_BEGIN); + visit(ForStmtContext.class).delegateTerminal(context -> context.block().R_CURLY()).map(FOR_BLOCK_END); + + visit(SwitchStmtContext.class).map(SWITCH_STATEMENT); + visit(ExprSwitchStmtContext.class).delegateTerminal(ExprSwitchStmtContext::L_CURLY).map(SWITCH_BLOCK_BEGIN); + visit(TypeSwitchStmtContext.class).delegateTerminal(TypeSwitchStmtContext::L_CURLY).map(SWITCH_BLOCK_BEGIN); + visit(ExprSwitchStmtContext.class).delegateTerminalExit(ExprSwitchStmtContext::R_CURLY).map(SWITCH_BLOCK_END); + visit(TypeSwitchStmtContext.class).delegateTerminalExit(TypeSwitchStmtContext::R_CURLY).map(SWITCH_BLOCK_END); + + visit(ExprCaseClauseContext.class).map(SWITCH_CASE); + visit(ExprCaseClauseContext.class).delegateContext(ExprCaseClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + visit(TypeCaseClauseContext.class).map(SWITCH_CASE); + visit(TypeCaseClauseContext.class).delegateContext(TypeCaseClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + + visit(SelectStmtContext.class).map(SELECT_STATEMENT); + visit(SelectStmtContext.class).delegateTerminal(SelectStmtContext::L_CURLY).map(SELECT_BLOCK_BEGIN); + visit(SelectStmtContext.class).delegateTerminal(SelectStmtContext::R_CURLY).map(SELECT_BLOCK_END); + + visit(CommCaseContext.class).map(SWITCH_CASE); + visit(CommClauseContext.class).delegateContext(CommClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + } + + private void statements() { + visit(VarDeclContext.class).mapRange(VARIABLE_DECLARATION); + visit(ConstSpecContext.class).map(VARIABLE_DECLARATION); + + visit(FunctionLitContext.class).map(FUNCTION_LITERAL); + visit(FunctionLitContext.class).delegateContext(FunctionLitContext::block).map(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END); + + visit(AssignmentContext.class).mapRange(ASSIGNMENT); + + visit(ShortVarDeclContext.class).map(VARIABLE_DECLARATION); + visit(ShortVarDeclContext.class).map(ASSIGNMENT); + + visit(ExpressionContext.class, context -> hasAncestor(context, ArgumentsContext.class)).map(ARGUMENT); + + visit(StatementContext.class).map(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END); + } + + private void objectCreation() { + visit(KeyedElementContext.class, context -> hasAncestor(context, ArrayTypeContext.class) && + hasAncestor(context, CompositeLitContext.class)).mapRange(ARRAY_ELEMENT); + visit(KeyedElementContext.class, context -> hasAncestor(context, StructTypeContext.class)).mapRange(MEMBER_DECLARATION); + visit(KeyedElementContext.class, context -> hasAncestor(context, MapTypeContext.class) && + hasAncestor(context, CompositeLitContext.class)).mapRange(MAP_ELEMENT); + visit(KeyedElementContext.class, context -> hasAncestor(context, SliceTypeContext.class) && + hasAncestor(context, CompositeLitContext.class)).mapRange(SLICE_ELEMENT); + visit(KeyedElementContext.class, context -> hasAncestor(context, TypeNameContext.class) && + hasAncestor(context, CompositeLitContext.class)).mapRange(NAMED_TYPE_ELEMENT); + + visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(ARRAY_CONSTRUCTOR); + visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(ArrayTypeContext::L_BRACKET).map(ARRAY_BODY_BEGIN); + visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(ArrayTypeContext::R_BRACKET).map(ARRAY_BODY_END); + + visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(SLICE_CONSTRUCTOR); + visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(SliceTypeContext::L_BRACKET).map(SLICE_BODY_BEGIN); + visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(SliceTypeContext::R_BRACKET).map(SLICE_BODY_END); + + visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(MAP_CONSTRUCTOR); + visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(MapTypeContext::L_BRACKET).map(MAP_BODY_BEGIN); + visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(MapTypeContext::R_BRACKET).map(MAP_BODY_END); + + visit(TypeNameContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(NAMED_TYPE_CONSTRUCTOR); + visit(TypeNameContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateContext(context -> Objects.requireNonNull(getAncestor(context, CompositeLitContext.class)).literalValue()).map(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END); + visit(TypeNameContext.class, context -> hasAncestor(context, InterfaceTypeContext.class)).mapRange(TYPE_CONSTRAINT); + + visit(TypeAssertionContext.class).mapRange(TYPE_ASSERTION); + visit(MethodSpecContext.class).mapRange(INTERFACE_METHOD); + } + + private void controlFlowKeywords() { + visit(ReturnStmtContext.class).mapRange(RETURN); + visit(BreakStmtContext.class).mapRange(BREAK); + visit(ContinueStmtContext.class).mapRange(CONTINUE); + visit(FallthroughStmtContext.class).mapRange(FALLTHROUGH); + visit(GotoStmtContext.class).mapRange(GOTO); + visit(GotoStmtContext.class).mapRange(GO); + visit(DeferStmtContext.class).mapRange(DEFER); + visit(SendStmtContext.class).mapRange(SEND_STATEMENT); + visit(RecvStmtContext.class).mapRange(RECEIVE_STATEMENT); + } +} diff --git a/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java b/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java index 3d9da82c6..0f6a76621 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java @@ -1,61 +1,32 @@ package de.jplag.golang; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; - -import de.jplag.AbstractParser; -import de.jplag.ParsingException; -import de.jplag.Token; -import de.jplag.TokenType; +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.AbstractAntlrParserAdapter; import de.jplag.golang.grammar.GoLexer; import de.jplag.golang.grammar.GoParser; -import de.jplag.util.FileUtils; - -public class GoParserAdapter extends AbstractParser { - private File currentFile; - private List tokens; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.ParserRuleContext; - public List parse(Set files) throws ParsingException { - tokens = new ArrayList<>(); - for (File file : files) { - parseFile(file); - tokens.add(Token.fileEnd(file)); - } - return tokens; +public class GoParserAdapter extends AbstractAntlrParserAdapter { + @Override + protected Lexer createLexer(CharStream input) { + return new GoLexer(input); } - private void parseFile(File file) throws ParsingException { - try (BufferedReader reader = FileUtils.openFileReader(file)) { - currentFile = file; - - GoLexer lexer = new GoLexer(CharStreams.fromReader(reader)); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); - GoParser parser = new GoParser(tokenStream); - - ParserRuleContext entryContext = parser.sourceFile(); - ParseTreeWalker treeWalker = new ParseTreeWalker(); + @Override + protected GoParser createParser(CommonTokenStream tokenStream) { + return new GoParser(tokenStream); + } - JPlagGoListener listener = new JPlagGoListener(this); - for (int i = 0; i < entryContext.getChildCount(); i++) { - ParseTree parseTree = entryContext.getChild(i); - treeWalker.walk(listener, parseTree); - } - } catch (IOException exception) { - throw new ParsingException(file, exception.getMessage(), exception); - } + @Override + protected ParserRuleContext getEntryContext(GoParser parser) { + return parser.sourceFile(); } - public void addToken(TokenType tokenType, int line, int column, int length) { - tokens.add(new Token(tokenType, currentFile, line, column, length)); + @Override + protected AbstractAntlrListener getListener() { + return new GoListener(); } } diff --git a/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java b/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java deleted file mode 100644 index e2251353d..000000000 --- a/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java +++ /dev/null @@ -1,608 +0,0 @@ -package de.jplag.golang; - -import static de.jplag.golang.GoTokenType.ARGUMENT; -import static de.jplag.golang.GoTokenType.ARRAY_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.ARRAY_BODY_END; -import static de.jplag.golang.GoTokenType.ARRAY_CONSTRUCTOR; -import static de.jplag.golang.GoTokenType.ARRAY_ELEMENT; -import static de.jplag.golang.GoTokenType.ASSIGNMENT; -import static de.jplag.golang.GoTokenType.BREAK; -import static de.jplag.golang.GoTokenType.CASE_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.CASE_BLOCK_END; -import static de.jplag.golang.GoTokenType.CONTINUE; -import static de.jplag.golang.GoTokenType.DEFER; -import static de.jplag.golang.GoTokenType.ELSE_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.ELSE_BLOCK_END; -import static de.jplag.golang.GoTokenType.FALLTHROUGH; -import static de.jplag.golang.GoTokenType.FOR_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.FOR_BLOCK_END; -import static de.jplag.golang.GoTokenType.FOR_STATEMENT; -import static de.jplag.golang.GoTokenType.FUNCTION_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.FUNCTION_BODY_END; -import static de.jplag.golang.GoTokenType.FUNCTION_DECLARATION; -import static de.jplag.golang.GoTokenType.FUNCTION_LITERAL; -import static de.jplag.golang.GoTokenType.FUNCTION_PARAMETER; -import static de.jplag.golang.GoTokenType.GO; -import static de.jplag.golang.GoTokenType.GOTO; -import static de.jplag.golang.GoTokenType.IF_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.IF_BLOCK_END; -import static de.jplag.golang.GoTokenType.IF_STATEMENT; -import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE; -import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_BEGIN; -import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_END; -import static de.jplag.golang.GoTokenType.IMPORT_DECLARATION; -import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_END; -import static de.jplag.golang.GoTokenType.INTERFACE_DECLARATION; -import static de.jplag.golang.GoTokenType.INTERFACE_METHOD; -import static de.jplag.golang.GoTokenType.INVOCATION; -import static de.jplag.golang.GoTokenType.MAP_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.MAP_BODY_END; -import static de.jplag.golang.GoTokenType.MAP_CONSTRUCTOR; -import static de.jplag.golang.GoTokenType.MAP_ELEMENT; -import static de.jplag.golang.GoTokenType.MEMBER_DECLARATION; -import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_END; -import static de.jplag.golang.GoTokenType.NAMED_TYPE_CONSTRUCTOR; -import static de.jplag.golang.GoTokenType.NAMED_TYPE_ELEMENT; -import static de.jplag.golang.GoTokenType.PACKAGE; -import static de.jplag.golang.GoTokenType.RECEIVER; -import static de.jplag.golang.GoTokenType.RECEIVE_STATEMENT; -import static de.jplag.golang.GoTokenType.RETURN; -import static de.jplag.golang.GoTokenType.SELECT_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.SELECT_BLOCK_END; -import static de.jplag.golang.GoTokenType.SELECT_STATEMENT; -import static de.jplag.golang.GoTokenType.SEND_STATEMENT; -import static de.jplag.golang.GoTokenType.SLICE_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.SLICE_BODY_END; -import static de.jplag.golang.GoTokenType.SLICE_CONSTRUCTOR; -import static de.jplag.golang.GoTokenType.SLICE_ELEMENT; -import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_END; -import static de.jplag.golang.GoTokenType.STRUCT_BODY_BEGIN; -import static de.jplag.golang.GoTokenType.STRUCT_BODY_END; -import static de.jplag.golang.GoTokenType.STRUCT_DECLARATION; -import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_BEGIN; -import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_END; -import static de.jplag.golang.GoTokenType.SWITCH_CASE; -import static de.jplag.golang.GoTokenType.SWITCH_STATEMENT; -import static de.jplag.golang.GoTokenType.TYPE_ASSERTION; -import static de.jplag.golang.GoTokenType.TYPE_CONSTRAINT; -import static de.jplag.golang.GoTokenType.VARIABLE_DECLARATION; - -import java.util.Arrays; -import java.util.Deque; -import java.util.LinkedList; -import java.util.Optional; - -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.tree.TerminalNode; - -import de.jplag.TokenType; -import de.jplag.golang.grammar.GoParser; -import de.jplag.golang.grammar.GoParserBaseListener; - -public class JPlagGoListener extends GoParserBaseListener { - - private final GoParserAdapter parserAdapter; - private final Deque blockContexts; - - public JPlagGoListener(GoParserAdapter parserAdapter) { - this.parserAdapter = parserAdapter; - blockContexts = new LinkedList<>(); - } - - /** - * Passes a token of the given tokenType to the parserAdapter, representing the grammar's token given by token. - * @param tokenType the custom token type that occurred. - * @param token the corresponding grammar's token - */ - private void transformToken(TokenType tokenType, Token token) { - parserAdapter.addToken(tokenType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length()); - } - - /** - * Passes a token of the given tokenType to the parserAdapter, representing the current grammatical context given by - * start and end. - * @param tokenType the custom token type that occurred. - * @param start the first Token of the context - * @param end the last Token of the context - */ - private void transformToken(GoTokenType tokenType, Token start, Token end) { - parserAdapter.addToken(tokenType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1); - } - - private void enterContext(GoBlockContext context) { - blockContexts.push(context); - } - - private void expectAndLeave(GoBlockContext... contexts) { - GoBlockContext topContext = blockContexts.pop(); - assert Arrays.stream(contexts).anyMatch(context -> context == topContext); - } - - private GoBlockContext getCurrentContext() { - return blockContexts.peek(); - } - - /* TOP LEVEL STRUCTURES */ - - @Override - public void enterPackageClause(GoParser.PackageClauseContext context) { - transformToken(PACKAGE, context.getStart(), context.getStop()); - super.enterPackageClause(context); - } - - @Override - public void enterImportDecl(GoParser.ImportDeclContext context) { - transformToken(IMPORT_DECLARATION, context.getStart()); - - // if the children contain TerminalNodes, then it must be '(' and ')' - Optional listStart = context.children.stream().filter(TerminalNode.class::isInstance).map(TerminalNode.class::cast).findFirst(); - listStart.ifPresent(lParenTree -> transformToken(IMPORT_CLAUSE_BEGIN, lParenTree.getSymbol())); - - super.enterImportDecl(context); - } - - @Override - public void exitImportDecl(GoParser.ImportDeclContext context) { - if (context.getStop().getText().equals(")")) { - transformToken(IMPORT_CLAUSE_END, context.getStop()); - } - super.exitImportDecl(context); - } - - @Override - public void enterImportSpec(GoParser.ImportSpecContext context) { - transformToken(IMPORT_CLAUSE, context.getStart(), context.getStop()); - super.enterImportSpec(context); - } - - /* INTERFACE */ - - @Override - public void enterInterfaceType(GoParser.InterfaceTypeContext context) { - transformToken(INTERFACE_DECLARATION, context.getStart()); - enterContext(GoBlockContext.INTERFACE_BODY); - super.enterInterfaceType(context); - } - - @Override - public void exitInterfaceType(GoParser.InterfaceTypeContext context) { - expectAndLeave(GoBlockContext.INTERFACE_BODY); - super.exitInterfaceType(context); - } - - /* STRUCT */ - - @Override - public void enterStructType(GoParser.StructTypeContext context) { - transformToken(STRUCT_DECLARATION, context.getStart()); - enterContext(GoBlockContext.STRUCT_BODY); - super.enterStructType(context); - } - - @Override - public void exitStructType(GoParser.StructTypeContext context) { - expectAndLeave(GoBlockContext.STRUCT_BODY); - super.exitStructType(context); - } - - @Override - public void enterFieldDecl(GoParser.FieldDeclContext context) { - transformToken(MEMBER_DECLARATION, context.getStart(), context.getStop()); - super.enterFieldDecl(context); - } - - /* FUNCTION */ - - @Override - public void enterFunctionDecl(GoParser.FunctionDeclContext context) { - transformToken(FUNCTION_DECLARATION, context.getStart()); - enterContext(GoBlockContext.FUNCTION_BODY); - super.enterFunctionDecl(context); - } - - @Override - public void exitFunctionDecl(GoParser.FunctionDeclContext context) { - expectAndLeave(GoBlockContext.FUNCTION_BODY); - super.exitFunctionDecl(context); - } - - @Override - public void enterMethodDecl(GoParser.MethodDeclContext context) { - transformToken(FUNCTION_DECLARATION, context.getStart()); - enterContext(GoBlockContext.FUNCTION_BODY); - super.enterMethodDecl(context); - } - - @Override - public void exitMethodDecl(GoParser.MethodDeclContext context) { - expectAndLeave(GoBlockContext.FUNCTION_BODY); - super.exitMethodDecl(context); - } - - @Override - public void enterParameterDecl(GoParser.ParameterDeclContext context) { - if (context.parent.parent instanceof GoParser.ReceiverContext) { - transformToken(RECEIVER, context.getStart(), context.getStop()); - } else { - transformToken(FUNCTION_PARAMETER, context.getStart(), context.getStop()); - } - super.enterParameterDecl(context); - } - - /* CONTROL FLOW STATEMENTS */ - - @Override - public void enterIfStmt(GoParser.IfStmtContext context) { - transformToken(IF_STATEMENT, context.getStart()); - enterContext(GoBlockContext.IF_BLOCK); - super.enterIfStmt(context); - } - - @Override - public void exitIfStmt(GoParser.IfStmtContext context) { - expectAndLeave(GoBlockContext.IF_BLOCK, GoBlockContext.ELSE_BLOCK); - super.exitIfStmt(context); - } - - @Override - public void enterForStmt(GoParser.ForStmtContext context) { - transformToken(FOR_STATEMENT, context.getStart()); - enterContext(GoBlockContext.FOR_BLOCK); - super.enterForStmt(context); - } - - @Override - public void exitForStmt(GoParser.ForStmtContext context) { - expectAndLeave(GoBlockContext.FOR_BLOCK); - super.exitForStmt(context); - } - - @Override - public void enterSwitchStmt(GoParser.SwitchStmtContext context) { - transformToken(SWITCH_STATEMENT, context.getStart()); - enterContext(GoBlockContext.SWITCH_BLOCK); - super.enterSwitchStmt(context); - } - - @Override - public void exitSwitchStmt(GoParser.SwitchStmtContext context) { - expectAndLeave(GoBlockContext.SWITCH_BLOCK); - super.exitSwitchStmt(context); - } - - @Override - public void enterExprCaseClause(GoParser.ExprCaseClauseContext context) { - transformToken(SWITCH_CASE, context.getStart()); - var caseBlock = context.getChild(GoParser.StatementListContext.class, 0); - if (caseBlock != null) { - enterContext(GoBlockContext.CASE_BLOCK); - transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart()); - } - super.enterExprCaseClause(context); - } - - @Override - public void exitExprCaseClause(GoParser.ExprCaseClauseContext context) { - if (getCurrentContext() == GoBlockContext.CASE_BLOCK) { - transformToken(CASE_BLOCK_END, context.getStop()); - expectAndLeave(GoBlockContext.CASE_BLOCK); - } - super.exitExprCaseClause(context); - } - - @Override - public void enterTypeCaseClause(GoParser.TypeCaseClauseContext context) { - transformToken(SWITCH_CASE, context.getStart()); - var caseBlock = context.getChild(GoParser.StatementListContext.class, 0); - if (caseBlock != null) { - enterContext(GoBlockContext.CASE_BLOCK); - transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart()); - } - super.enterTypeCaseClause(context); - } - - @Override - public void exitTypeCaseClause(GoParser.TypeCaseClauseContext context) { - if (getCurrentContext() == GoBlockContext.CASE_BLOCK) { - transformToken(CASE_BLOCK_END, context.getStop()); - expectAndLeave(GoBlockContext.CASE_BLOCK); - } - super.exitTypeCaseClause(context); - } - - @Override - public void enterSelectStmt(GoParser.SelectStmtContext context) { - transformToken(SELECT_STATEMENT, context.getStart()); - enterContext(GoBlockContext.SELECT_CONTEXT); - super.enterSelectStmt(context); - } - - @Override - public void exitSelectStmt(GoParser.SelectStmtContext context) { - expectAndLeave(GoBlockContext.SELECT_CONTEXT); - super.exitSelectStmt(context); - } - - @Override - public void enterCommCase(GoParser.CommCaseContext context) { - transformToken(SWITCH_CASE, context.getStart()); - var caseBlock = context.getChild(GoParser.StatementListContext.class, 0); - if (caseBlock != null) { - enterContext(GoBlockContext.CASE_BLOCK); - transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart()); - } - super.enterCommCase(context); - } - - @Override - public void exitCommCase(GoParser.CommCaseContext context) { - if (getCurrentContext() == GoBlockContext.CASE_BLOCK) { - transformToken(CASE_BLOCK_END, context.getStop()); - expectAndLeave(GoBlockContext.CASE_BLOCK); - } - super.exitCommCase(context); - } - - /* STATEMENTS */ - - @Override - public void enterVarDecl(GoParser.VarDeclContext context) { - transformToken(VARIABLE_DECLARATION, context.getStart(), context.getStop()); - super.enterVarDecl(context); - } - - @Override - public void enterConstSpec(GoParser.ConstSpecContext context) { - transformToken(VARIABLE_DECLARATION, context.getStart()); - super.enterConstSpec(context); - } - - @Override - public void enterFunctionLit(GoParser.FunctionLitContext context) { - transformToken(FUNCTION_LITERAL, context.getStart()); - enterContext(GoBlockContext.FUNCTION_BODY); - super.enterFunctionLit(context); - } - - @Override - public void exitFunctionLit(GoParser.FunctionLitContext context) { - expectAndLeave(GoBlockContext.FUNCTION_BODY); - super.exitFunctionLit(context); - } - - @Override - public void enterAssignment(GoParser.AssignmentContext context) { - transformToken(ASSIGNMENT, context.getStart(), context.getStop()); - super.enterAssignment(context); - } - - @Override - public void enterShortVarDecl(GoParser.ShortVarDeclContext context) { - transformToken(VARIABLE_DECLARATION, context.getStart()); - transformToken(ASSIGNMENT, context.getStart()); - super.enterShortVarDecl(context); - } - - @Override - public void enterArguments(GoParser.ArgumentsContext context) { - transformToken(INVOCATION, context.getStart(), context.getStop()); - - // Arguments consist of ExpressionLists, which consist of Expressions - // Get all Expressions of all ExpressionLists in this ArgumentsContext - context.getRuleContexts(GoParser.ExpressionListContext.class).stream() - .flatMap(child -> child.getRuleContexts(GoParser.ExpressionContext.class).stream()) - .forEachOrdered(arg -> transformToken(ARGUMENT, arg.getStart(), arg.getStop())); - super.enterArguments(context); - } - - @Override - public void enterStatement(GoParser.StatementContext context) { - enterContext(GoBlockContext.STATEMENT_BLOCK); - super.enterStatement(context); - } - - @Override - public void exitStatement(GoParser.StatementContext context) { - expectAndLeave(GoBlockContext.STATEMENT_BLOCK); - super.exitStatement(context); - } - - /* OBJECT CREATION */ - - @Override - public void enterKeyedElement(GoParser.KeyedElementContext context) { - Optional tokenType = getCurrentContext().getElement(); - tokenType.ifPresent(type -> transformToken(type, context.getStart(), context.getStop())); - super.enterKeyedElement(context); - } - - @Override - public void enterArrayType(GoParser.ArrayTypeContext context) { - // otherwise, it is just a type expression - if (context.parent.parent instanceof GoParser.CompositeLitContext) { - enterContext(GoBlockContext.ARRAY_BODY); - transformToken(ARRAY_CONSTRUCTOR, context.getStart(), context.getStop()); - } - super.enterArrayType(context); - } - - @Override - public void enterSliceType(GoParser.SliceTypeContext context) { - // otherwise, it is just a type expression - if (context.parent.parent instanceof GoParser.CompositeLitContext) { - enterContext(GoBlockContext.SLICE_BODY); - transformToken(SLICE_CONSTRUCTOR, context.getStart(), context.getStop()); - } - super.enterSliceType(context); - } - - @Override - public void exitCompositeLit(GoParser.CompositeLitContext context) { - expectAndLeave(GoBlockContext.MAP_BODY, GoBlockContext.SLICE_BODY, GoBlockContext.ARRAY_BODY, GoBlockContext.NAMED_TYPE_BODY); - super.exitCompositeLit(context); - } - - @Override - public void enterMapType(GoParser.MapTypeContext context) { - // otherwise, it is just a type expression - if (context.parent.parent instanceof GoParser.CompositeLitContext) { - enterContext(GoBlockContext.MAP_BODY); - transformToken(MAP_CONSTRUCTOR, context.getStart(), context.getStop()); - } - super.enterMapType(context); - } - - @Override - public void enterTypeName(GoParser.TypeNameContext context) { - if (context.parent.parent instanceof GoParser.CompositeLitContext) { - transformToken(NAMED_TYPE_CONSTRUCTOR, context.getStart()); - enterContext(GoBlockContext.NAMED_TYPE_BODY); - } else if (context.parent instanceof GoParser.InterfaceTypeContext) { - transformToken(TYPE_CONSTRAINT, context.getStart(), context.getStop()); - } - super.enterTypeName(context); - } - - @Override - public void enterTypeAssertion(GoParser.TypeAssertionContext context) { - transformToken(TYPE_ASSERTION, context.getStart(), context.getStop()); - super.enterTypeAssertion(context); - } - - @Override - public void enterMethodSpec(GoParser.MethodSpecContext context) { - transformToken(INTERFACE_METHOD, context.getStart(), context.getStop()); - super.enterMethodSpec(context); - } - - /* CONTROL FLOW KEYWORDS */ - - @Override - public void enterReturnStmt(GoParser.ReturnStmtContext context) { - transformToken(RETURN, context.getStart(), context.getStop()); - super.enterReturnStmt(context); - } - - @Override - public void enterBreakStmt(GoParser.BreakStmtContext context) { - transformToken(BREAK, context.getStart(), context.getStop()); - super.enterBreakStmt(context); - } - - @Override - public void enterContinueStmt(GoParser.ContinueStmtContext context) { - transformToken(CONTINUE, context.getStart(), context.getStop()); - super.enterContinueStmt(context); - } - - @Override - public void enterFallthroughStmt(GoParser.FallthroughStmtContext context) { - transformToken(FALLTHROUGH, context.getStart(), context.getStop()); - super.enterFallthroughStmt(context); - } - - @Override - public void enterGotoStmt(GoParser.GotoStmtContext context) { - transformToken(GOTO, context.getStart(), context.getStop()); - super.enterGotoStmt(context); - } - - @Override - public void enterGoStmt(GoParser.GoStmtContext context) { - transformToken(GO, context.getStart(), context.getStop()); - super.enterGoStmt(context); - } - - @Override - public void enterDeferStmt(GoParser.DeferStmtContext context) { - transformToken(DEFER, context.getStart(), context.getStop()); - super.enterDeferStmt(context); - } - - @Override - public void enterSendStmt(GoParser.SendStmtContext ctx) { - transformToken(SEND_STATEMENT, ctx.getStart(), ctx.getStop()); - super.enterSendStmt(ctx); - } - - @Override - public void enterRecvStmt(GoParser.RecvStmtContext ctx) { - transformToken(RECEIVE_STATEMENT, ctx.getStart(), ctx.getStop()); - super.enterRecvStmt(ctx); - } - - @Override - public void visitTerminal(TerminalNode node) { - Token token = node.getSymbol(); - switch (token.getText()) { - case "else" -> { - expectAndLeave(GoBlockContext.IF_BLOCK); - enterContext(GoBlockContext.ELSE_BLOCK); - } - case "{" -> { - if (getCurrentContext() != null) { - transformToken(getCurrentContext().getBegin(), token); - } - } - case "}" -> { - if (getCurrentContext() != null) { - transformToken(getCurrentContext().getEnd(), token); - } - } - default -> { - // do nothing. - } - } - super.visitTerminal(node); - } - - /** - * This enumeration provides sets of information regarding different types of nesting structures in Go. Each element is - * a tuple of a token for the beginning of a block, the end of the block, and optionally, for the elements contained. - *

- * As the Go parser does not differentiate between different kinds of blocks, a stack of these GoBlockContexts is - * required to be able to assign the correct token types for each block. - */ - private enum GoBlockContext { - ARRAY_BODY(ARRAY_BODY_BEGIN, ARRAY_BODY_END, Optional.of(ARRAY_ELEMENT)), - STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END, Optional.of(MEMBER_DECLARATION)), - MAP_BODY(MAP_BODY_BEGIN, MAP_BODY_END, Optional.of(MAP_ELEMENT)), - SLICE_BODY(SLICE_BODY_BEGIN, SLICE_BODY_END, Optional.of(SLICE_ELEMENT)), - NAMED_TYPE_BODY(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END, Optional.of(NAMED_TYPE_ELEMENT)), - FUNCTION_BODY(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END, Optional.empty()), - - IF_BLOCK(IF_BLOCK_BEGIN, IF_BLOCK_END, Optional.empty()), - ELSE_BLOCK(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END, Optional.empty()), - FOR_BLOCK(FOR_BLOCK_BEGIN, FOR_BLOCK_END, Optional.empty()), - SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END, Optional.empty()), - SELECT_CONTEXT(SELECT_BLOCK_BEGIN, SELECT_BLOCK_END, Optional.empty()), - STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END, Optional.empty()), - CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, Optional.empty()), - INTERFACE_BODY(INTERFACE_BLOCK_BEGIN, INTERFACE_BLOCK_END, Optional.empty()); - - private final GoTokenType beginTokenType; - private final GoTokenType endTokenType; - private final Optional elementTokenType; - - GoBlockContext(GoTokenType beginTokenType, GoTokenType endTokenType, Optional elementTokenType) { - this.beginTokenType = beginTokenType; - this.endTokenType = endTokenType; - this.elementTokenType = elementTokenType; - } - - GoTokenType getBegin() { - return this.beginTokenType; - } - - GoTokenType getEnd() { - return this.endTokenType; - } - - public Optional getElement() { - return this.elementTokenType; - } - } -} From c019c9668ab44bcee570899aa0a4a1f97babcaf6 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 9 Apr 2024 14:57:50 +0200 Subject: [PATCH 2/7] Fixed errors in the new go listener and migrated the test to the new framework --- .../java/de/jplag/antlr/ContextVisitor.java | 2 +- .../java/de/jplag/antlr/TokenCollector.java | 2 +- .../main/java/de/jplag/golang/GoListener.java | 100 +++++++----- .../java/de/jplag/golang/GoLanguageTest.java | 153 ++---------------- .../de/jplag/{golang => go}/Complete.go | 0 5 files changed, 72 insertions(+), 185 deletions(-) rename languages/golang/src/test/resources/de/jplag/{golang => go}/Complete.go (100%) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index dd86ba85d..dbc6e8bf1 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -114,7 +114,7 @@ public TerminalVisitor delegateTerminalExit(Function mapper) { */ public ContextVisitor delegateContext(Function mapper) { ContextVisitor visitor = new ContextVisitor<>((ignore) -> true); - this.delegate = new DelegateVisitor<>(visitor, mapper); + this.delegate = new ContextDelegateVisitor<>(visitor, mapper); return visitor; } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index 588c471b2..c1d7e1317 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -48,7 +48,7 @@ void addToken(TokenType jplagType, Function semanticsSuppl org.antlr.v4.runtime.Token antlrEndToken = extractEndToken.apply(entity); int line = antlrToken.getLine(); int column = antlrToken.getCharPositionInLine() + 1; - int length = (antlrEndToken.getStartIndex() - antlrToken.getStartIndex()) + antlrEndToken.getText().length(); + int length = (antlrEndToken.getStopIndex() - antlrToken.getStartIndex()) + 1; Token token; if (extractsSemantics) { if (semanticsSupplier == null) { diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java index a67125e6c..2e9770868 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -1,9 +1,11 @@ package de.jplag.golang; import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.ContextVisitor; import de.jplag.golang.grammar.GoParser.*; +import org.antlr.v4.runtime.ParserRuleContext; -import java.util.Objects; +import java.util.function.Function; import static de.jplag.golang.GoTokenType.*; @@ -48,11 +50,11 @@ private void structDeclarations() { } private void functionDeclarations() { - visit(FunctionDeclContext.class).map(FUNCTION_DECLARATION); + visit(FunctionDeclContext.class).delegateTerminal(FunctionDeclContext::FUNC).map(FUNCTION_DECLARATION); visit(FunctionDeclContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FUNCTION_BODY_BEGIN); visit(FunctionDeclContext.class).delegateTerminalExit(context -> context.block().R_CURLY()).map(FUNCTION_BODY_END); - visit(MethodDeclContext.class).map(FUNCTION_DECLARATION); + visit(MethodDeclContext.class).delegateTerminal(MethodDeclContext::FUNC).map(FUNCTION_DECLARATION); visit(MethodDeclContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FUNCTION_BODY_BEGIN); visit(MethodDeclContext.class).delegateTerminalExit(context -> context.block().R_CURLY()).map(FUNCTION_BODY_END); @@ -61,16 +63,14 @@ private void functionDeclarations() { } private void controlFlowRules() { - visit(IfStmtContext.class).map(IF_STATEMENT); - visit(IfStmtContext.class).delegateTerminal(context -> context.block(0).L_CURLY()).map(IF_BLOCK_BEGIN); - visit(IfStmtContext.class).delegateTerminal(context -> context.block(0).R_CURLY()).map(IF_BLOCK_END); + visit(IfStmtContext.class).delegateTerminal(IfStmtContext::IF).map(IF_STATEMENT); + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext)context.parent).block(0))).map(IF_BLOCK_BEGIN, IF_BLOCK_END); //TODO no else token? - visit(IfStmtContext.class, context -> context.ELSE() != null).delegateTerminal(context -> context.block(1).L_CURLY()).map(ELSE_BLOCK_BEGIN); - visit(IfStmtContext.class, context -> context.ELSE() != null).delegateTerminal(context -> context.block(1).L_CURLY()).map(ELSE_BLOCK_END); + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext)context.parent).block(1))).map(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END); visit(ForStmtContext.class).map(FOR_STATEMENT); visit(ForStmtContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FOR_BLOCK_BEGIN); - visit(ForStmtContext.class).delegateTerminal(context -> context.block().R_CURLY()).map(FOR_BLOCK_END); + visit(ForStmtContext.class).delegateTerminalExit(context -> context.block().R_CURLY()).map(FOR_BLOCK_END); visit(SwitchStmtContext.class).map(SWITCH_STATEMENT); visit(ExprSwitchStmtContext.class).delegateTerminal(ExprSwitchStmtContext::L_CURLY).map(SWITCH_BLOCK_BEGIN); @@ -79,16 +79,16 @@ private void controlFlowRules() { visit(TypeSwitchStmtContext.class).delegateTerminalExit(TypeSwitchStmtContext::R_CURLY).map(SWITCH_BLOCK_END); visit(ExprCaseClauseContext.class).map(SWITCH_CASE); - visit(ExprCaseClauseContext.class).delegateContext(ExprCaseClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + visit(StatementListContext.class, context -> context.parent instanceof ExprCaseClauseContext).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); visit(TypeCaseClauseContext.class).map(SWITCH_CASE); - visit(TypeCaseClauseContext.class).delegateContext(TypeCaseClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + visit(StatementListContext.class, context -> context.parent instanceof TypeCaseClauseContext).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); visit(SelectStmtContext.class).map(SELECT_STATEMENT); visit(SelectStmtContext.class).delegateTerminal(SelectStmtContext::L_CURLY).map(SELECT_BLOCK_BEGIN); - visit(SelectStmtContext.class).delegateTerminal(SelectStmtContext::R_CURLY).map(SELECT_BLOCK_END); + visit(SelectStmtContext.class).delegateTerminalExit(SelectStmtContext::R_CURLY).map(SELECT_BLOCK_END); visit(CommCaseContext.class).map(SWITCH_CASE); - visit(CommClauseContext.class).delegateContext(CommClauseContext::statementList).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); + visit(StatementListContext.class, context -> context.parent instanceof CommClauseContext).map(CASE_BLOCK_BEGIN, CASE_BLOCK_END); } private void statements() { @@ -103,37 +103,31 @@ private void statements() { visit(ShortVarDeclContext.class).map(VARIABLE_DECLARATION); visit(ShortVarDeclContext.class).map(ASSIGNMENT); - visit(ExpressionContext.class, context -> hasAncestor(context, ArgumentsContext.class)).map(ARGUMENT); + visit(ArgumentsContext.class).mapRange(INVOCATION); + visit(ExpressionContext.class, context -> hasAncestor(context, ArgumentsContext.class)).mapRange(ARGUMENT); - visit(StatementContext.class).map(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END); + visit(StatementContext.class).delegateContext(StatementContext::block).map(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END); } private void objectCreation() { - visit(KeyedElementContext.class, context -> hasAncestor(context, ArrayTypeContext.class) && - hasAncestor(context, CompositeLitContext.class)).mapRange(ARRAY_ELEMENT); - visit(KeyedElementContext.class, context -> hasAncestor(context, StructTypeContext.class)).mapRange(MEMBER_DECLARATION); - visit(KeyedElementContext.class, context -> hasAncestor(context, MapTypeContext.class) && - hasAncestor(context, CompositeLitContext.class)).mapRange(MAP_ELEMENT); - visit(KeyedElementContext.class, context -> hasAncestor(context, SliceTypeContext.class) && - hasAncestor(context, CompositeLitContext.class)).mapRange(SLICE_ELEMENT); - visit(KeyedElementContext.class, context -> hasAncestor(context, TypeNameContext.class) && - hasAncestor(context, CompositeLitContext.class)).mapRange(NAMED_TYPE_ELEMENT); - - visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(ARRAY_CONSTRUCTOR); - visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(ArrayTypeContext::L_BRACKET).map(ARRAY_BODY_BEGIN); - visit(ArrayTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(ArrayTypeContext::R_BRACKET).map(ARRAY_BODY_END); - - visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(SLICE_CONSTRUCTOR); - visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(SliceTypeContext::L_BRACKET).map(SLICE_BODY_BEGIN); - visit(SliceTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(SliceTypeContext::R_BRACKET).map(SLICE_BODY_END); - - visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(MAP_CONSTRUCTOR); - visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminal(MapTypeContext::L_BRACKET).map(MAP_BODY_BEGIN); - visit(MapTypeContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateTerminalExit(MapTypeContext::R_BRACKET).map(MAP_BODY_END); - - visit(TypeNameContext.class, context -> hasAncestor(context, CompositeLitContext.class)).map(NAMED_TYPE_CONSTRUCTOR); - visit(TypeNameContext.class, context -> hasAncestor(context, CompositeLitContext.class)).delegateContext(context -> Objects.requireNonNull(getAncestor(context, CompositeLitContext.class)).literalValue()).map(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END); - visit(TypeNameContext.class, context -> hasAncestor(context, InterfaceTypeContext.class)).mapRange(TYPE_CONSTRAINT); + visitKeyedElement(LiteralTypeContext::arrayType).mapRange(ARRAY_ELEMENT); + visitKeyedElement(LiteralTypeContext::structType).mapRange(MEMBER_DECLARATION); + visitKeyedElement(LiteralTypeContext::mapType).mapRange(MAP_ELEMENT); + visitKeyedElement(LiteralTypeContext::sliceType).mapRange(SLICE_ELEMENT); + visitKeyedElement(LiteralTypeContext::typeName).mapRange(NAMED_TYPE_ELEMENT); + + visitCompositeLitChild(ArrayTypeContext.class).map(ARRAY_CONSTRUCTOR); + visitCompositeLitDelegate(ArrayTypeContext.class).map(ARRAY_BODY_BEGIN, ARRAY_BODY_END); + + visitCompositeLitChild(SliceTypeContext.class).map(SLICE_CONSTRUCTOR); + visitCompositeLitDelegate(SliceTypeContext.class).map(SLICE_BODY_BEGIN, SLICE_BODY_END); + + visitCompositeLitChild(MapTypeContext.class).map(MAP_CONSTRUCTOR); + visitCompositeLitDelegate(MapTypeContext.class).map(MAP_BODY_BEGIN, MAP_BODY_END); + + visitCompositeLitChild(TypeNameContext.class).map(NAMED_TYPE_CONSTRUCTOR); + visitCompositeLitDelegate(TypeNameContext.class).map(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END); + visit(TypeNameContext.class, context -> context.parent instanceof InterfaceTypeContext).mapRange(TYPE_CONSTRAINT); visit(TypeAssertionContext.class).mapRange(TYPE_ASSERTION); visit(MethodSpecContext.class).mapRange(INTERFACE_METHOD); @@ -145,9 +139,33 @@ private void controlFlowKeywords() { visit(ContinueStmtContext.class).mapRange(CONTINUE); visit(FallthroughStmtContext.class).mapRange(FALLTHROUGH); visit(GotoStmtContext.class).mapRange(GOTO); - visit(GotoStmtContext.class).mapRange(GO); + visit(GoStmtContext.class).mapRange(GO); visit(DeferStmtContext.class).mapRange(DEFER); visit(SendStmtContext.class).mapRange(SEND_STATEMENT); visit(RecvStmtContext.class).mapRange(RECEIVE_STATEMENT); } + + private de.jplag.antlr.ContextVisitor visitCompositeLitChild(Class type) { + return visit(type, context -> context.parent.parent instanceof CompositeLitContext); + } + + private ContextVisitor visitCompositeLitDelegate(Class type) { + return visit(CompositeLitContext.class, context -> context.literalType().children.stream().anyMatch(it -> type.isAssignableFrom(it.getClass()))); + } + + private ContextVisitor visitKeyedElement(Function typeGetter) { + return visit(KeyedElementContext.class, context -> { + CompositeLitContext parent = getAncestor(context, CompositeLitContext.class); + if(parent == null) { + return false; + } + + LiteralTypeContext typeContext = parent.literalType(); + if(typeContext == null) { + return false; + } + + return typeGetter.apply(typeContext) != null; + }); + } } diff --git a/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java b/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java index 0feafc7bc..fc33d5561 100644 --- a/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java +++ b/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java @@ -1,156 +1,25 @@ package de.jplag.golang; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import de.jplag.testutils.LanguageModuleTest; +import de.jplag.testutils.datacollector.TestDataCollector; +import de.jplag.testutils.datacollector.TestSourceIgnoredLinesCollector; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.OptionalInt; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import de.jplag.ParsingException; -import de.jplag.SharedTokenType; -import de.jplag.Token; -import de.jplag.TokenPrinter; - -class GoLanguageTest { +class GoLanguageTest extends LanguageModuleTest { /** * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens. */ private static final String COMPLETE_TEST_FILE = "Complete.go"; - /** - * Regular expression that describes lines consisting only of whitespace and optionally a line comment. - */ - private static final String EMPTY_OR_SINGLE_LINE_COMMENT = "\\s*(//.*|/\\*.*\\*/)?"; - - /** - * Regular expression that describes lines containing the start of a multiline comment and no code before it. - */ - private static final String DELIMITED_COMMENT_START = "\\s*/\\*(?:(?!\\*/).)*$"; - - /** - * Regular expression that describes lines containing the end of a multiline comment and no more code after that. - */ - private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$"; - - private final Logger logger = LoggerFactory.getLogger(GoLanguageTest.class); - private final String[] testFiles = new String[] {COMPLETE_TEST_FILE}; - private final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "golang").toFile(); - private GoLanguage language; - - @BeforeEach - void setup() { - language = new GoLanguage(); + public GoLanguageTest() { + super(new GoLanguage(), GoTokenType.class); } - @Test - void parseTestFiles() throws ParsingException { - for (String fileName : testFiles) { - List tokens = language.parse(Set.of(new File(testFileLocation, fileName))); - String output = TokenPrinter.printTokens(tokens, testFileLocation); - logger.info(output); - - testSourceCoverage(fileName, tokens); - if (fileName.equals(COMPLETE_TEST_FILE)) { - testTokenCoverage(tokens, fileName); - } - } + @Override + protected void collectTestData(TestDataCollector collector) { + collector.testFile(COMPLETE_TEST_FILE).testCoverages(); } - /** - * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token. - * @param fileName a code sample file name - * @param tokens the list of tokens generated from the sample - */ - private void testSourceCoverage(String fileName, List tokens) { - File testFile = new File(testFileLocation, fileName); - - List lines = null; - try { - lines = Files.readAllLines(testFile.toPath()); - } catch (IOException exception) { - logger.info("Error while reading test file %s".formatted(fileName), exception); - fail(); - } - - // All lines that contain code - var codeLines = getCodeLines(lines); - // All lines that contain a token - var tokenLines = tokens.stream().mapToInt(Token::getLine).distinct().boxed().toList(); - - if (codeLines.size() > tokenLines.size()) { - List missedLinesIndices = new ArrayList<>(codeLines); - missedLinesIndices.removeAll(tokenLines); - var missedLines = missedLinesIndices.stream().map(Object::toString).collect(Collectors.joining(", ")); - if (!missedLines.isBlank()) { - fail("Found lines in file '%s' that are not represented in the token list. \n\tMissed lines: %s".formatted(fileName, missedLines)); - } - } - OptionalInt differingLine = IntStream.range(0, codeLines.size()) - .dropWhile(index -> Objects.equals(codeLines.get(index), tokenLines.get(index))).findAny(); - differingLine.ifPresent( - i -> fail("Not all lines of code in '%s' are represented in tokens, starting with line %d.".formatted(fileName, codeLines.get(i)))); + @Override + protected void configureIgnoredLines(TestSourceIgnoredLinesCollector collector) { } - - /** - * Gets the line numbers of lines containing actual code, omitting empty lines and comment lines. - * @param lines lines of a code file - * @return a list of the line numbers of code lines - */ - private List getCodeLines(List lines) { - // This boxed boolean can be accessed from within the lambda method below - var state = new Object() { - boolean insideComment = false; - }; - - var codeLines = IntStream.rangeClosed(1, lines.size()).sequential().filter(idx -> { - String line = lines.get(idx - 1); - if (line.matches(EMPTY_OR_SINGLE_LINE_COMMENT)) { - return false; - } else if (line.matches(DELIMITED_COMMENT_START)) { - state.insideComment = true; - return false; - } else if (state.insideComment) { - // This fails if code follows after '*/'. If the code is formatted well, this should not happen. - if (line.matches(DELIMITED_COMMENT_END)) { - state.insideComment = false; - } - return false; - } - return true; - }); - - return codeLines.boxed().toList(); - - } - - /** - * Confirms that all Token types are 'reachable' with a complete code example. - * @param tokens list of tokens which is supposed to contain all types of tokens - * @param fileName The file name of the complete code example - */ - private void testTokenCoverage(List tokens, String fileName) { - var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet()); - assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END)); - var annotatedGoTokens = annotatedTokens.stream().filter(GoTokenType.class::isInstance).collect(Collectors.toSet()); - var allGoTokens = GoTokenType.values(); - var missingGoTokens = Arrays.stream(allGoTokens).filter(token -> !annotatedGoTokens.contains(token)).toList(); - assertTrue(missingGoTokens.isEmpty(), "The following go tokens are missing in the code example '%s':\n".formatted(fileName) - + String.join("\n", missingGoTokens.stream().map(GoTokenType::getDescription).toList())); - } - } diff --git a/languages/golang/src/test/resources/de/jplag/golang/Complete.go b/languages/golang/src/test/resources/de/jplag/go/Complete.go similarity index 100% rename from languages/golang/src/test/resources/de/jplag/golang/Complete.go rename to languages/golang/src/test/resources/de/jplag/go/Complete.go From e6faa8b024e18ceae293b293d039fc8344cb5dac Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 9 Apr 2024 15:07:10 +0200 Subject: [PATCH 3/7] Added some basic tests to the go module --- .../java/de/jplag/golang/GoLanguageTest.java | 29 +++++++++++++++++-- .../de/jplag/go/ArrayEllipsisDecls.go | 14 +++++++++ .../test/resources/de/jplag/go/Constants.go | 6 ++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 languages/golang/src/test/resources/de/jplag/go/ArrayEllipsisDecls.go create mode 100644 languages/golang/src/test/resources/de/jplag/go/Constants.go diff --git a/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java b/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java index fc33d5561..68d3553e6 100644 --- a/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java +++ b/languages/golang/src/test/java/de/jplag/golang/GoLanguageTest.java @@ -1,14 +1,31 @@ package de.jplag.golang; +import static de.jplag.golang.GoTokenType.ARGUMENT; +import static de.jplag.golang.GoTokenType.ASSIGNMENT; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_END; +import static de.jplag.golang.GoTokenType.FUNCTION_DECLARATION; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_BEGIN; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_END; +import static de.jplag.golang.GoTokenType.IMPORT_DECLARATION; +import static de.jplag.golang.GoTokenType.INVOCATION; +import static de.jplag.golang.GoTokenType.MEMBER_DECLARATION; +import static de.jplag.golang.GoTokenType.PACKAGE; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_END; +import static de.jplag.golang.GoTokenType.STRUCT_DECLARATION; +import static de.jplag.golang.GoTokenType.VARIABLE_DECLARATION; + import de.jplag.testutils.LanguageModuleTest; import de.jplag.testutils.datacollector.TestDataCollector; import de.jplag.testutils.datacollector.TestSourceIgnoredLinesCollector; class GoLanguageTest extends LanguageModuleTest { - /** - * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens. - */ private static final String COMPLETE_TEST_FILE = "Complete.go"; + // example files taken from antlr repo + private static final String CONSTANTS_TEST_FILE = "Constants.go"; + private static final String ARRAY_ELLIPSIS_DECLS_FILE = "ArrayEllipsisDecls.go"; public GoLanguageTest() { super(new GoLanguage(), GoTokenType.class); @@ -17,6 +34,12 @@ public GoLanguageTest() { @Override protected void collectTestData(TestDataCollector collector) { collector.testFile(COMPLETE_TEST_FILE).testCoverages(); + + // Some basic tests, so we have at least some idea if the listener was changed + collector.testFile(CONSTANTS_TEST_FILE).testTokenSequence(PACKAGE, VARIABLE_DECLARATION, VARIABLE_DECLARATION); + collector.testFile(ARRAY_ELLIPSIS_DECLS_FILE).testSourceCoverage().testTokenSequence(PACKAGE, IMPORT_DECLARATION, IMPORT_CLAUSE_BEGIN, + IMPORT_CLAUSE, IMPORT_CLAUSE_END, STRUCT_DECLARATION, STRUCT_BODY_BEGIN, MEMBER_DECLARATION, STRUCT_BODY_END, FUNCTION_DECLARATION, + FUNCTION_BODY_BEGIN, VARIABLE_DECLARATION, ASSIGNMENT, INVOCATION, ARGUMENT, ARGUMENT, FUNCTION_BODY_END); } @Override diff --git a/languages/golang/src/test/resources/de/jplag/go/ArrayEllipsisDecls.go b/languages/golang/src/test/resources/de/jplag/go/ArrayEllipsisDecls.go new file mode 100644 index 000000000..0a9b83bec --- /dev/null +++ b/languages/golang/src/test/resources/de/jplag/go/ArrayEllipsisDecls.go @@ -0,0 +1,14 @@ +package samples + +import ( + "fmt" +) + +type Custom struct { + string +} + +func ArrayEllipsisDecls() { + stooges := [...]Custom{{"Moe"}, {"Larry"}, {"Curly"}} // len(stooges) == 3 + fmt.Println("Stooges: ", stooges) +} \ No newline at end of file diff --git a/languages/golang/src/test/resources/de/jplag/go/Constants.go b/languages/golang/src/test/resources/de/jplag/go/Constants.go new file mode 100644 index 000000000..1a3dab287 --- /dev/null +++ b/languages/golang/src/test/resources/de/jplag/go/Constants.go @@ -0,0 +1,6 @@ +package A + +const ( + /*A*/ T = 1 << 0 + /*B*/ Ta = 1 << 1 +) \ No newline at end of file From a88ab4b51f9bb43cdad76938c435a55e8d0341c0 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 9 Apr 2024 15:10:32 +0200 Subject: [PATCH 4/7] Added some basic tests to the go module --- .../java/de/jplag/antlr/AbstractVisitor.java | 11 +- .../jplag/antlr/ContextDelegateVisitor.java | 6 +- .../java/de/jplag/antlr/ContextVisitor.java | 31 ++-- .../java/de/jplag/antlr/DelegateVisitor.java | 2 +- .../main/java/de/jplag/antlr/HandlerData.java | 4 +- .../java/de/jplag/antlr/TokenCollector.java | 3 +- .../main/java/de/jplag/golang/GoLanguage.java | 3 +- .../main/java/de/jplag/golang/GoListener.java | 141 ++++++++++++++++-- .../java/de/jplag/golang/GoParserAdapter.java | 9 +- 9 files changed, 157 insertions(+), 53 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index c3ccafa53..17e8cb4b7 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -18,7 +18,6 @@ /** * The abstract visitor. - * * @param The type of the visited entity. */ public abstract class AbstractVisitor { @@ -39,7 +38,6 @@ public abstract class AbstractVisitor { /** * Add an action the visitor runs upon entering the entity. - * * @param handler The action, takes the entity and the variable registry as parameter. * @return Self */ @@ -50,7 +48,6 @@ public AbstractVisitor onEnter(BiConsumer handler) { /** * Add an action the visitor runs upon entering the entity. - * * @param handler The action, takes the entity as parameter. * @return Self */ @@ -61,7 +58,6 @@ public AbstractVisitor onEnter(Consumer handler) { /** * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. - * * @param tokenType The type of the token. * @return Self */ @@ -73,7 +69,6 @@ public AbstractVisitor mapEnter(TokenType tokenType) { /** * Tell the visitor that it should generate a token upon entering the entity. Should only be invoked once per visitor. * Alias for {@link #mapEnter(TokenType)}. - * * @param tokenType The type of the token. * @return Self */ @@ -84,7 +79,6 @@ public AbstractVisitor map(TokenType tokenType) { /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. - * * @param semanticsSupplier A function that takes the entity and returns the semantics. * @return Self */ @@ -95,7 +89,6 @@ public AbstractVisitor withSemantics(Function semanticsSupp /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics. - * * @param semanticsSupplier A function that returns the semantics. * @return Self */ @@ -106,7 +99,6 @@ public AbstractVisitor withSemantics(Supplier semanticsSupplie /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type control. - * * @return Self */ public AbstractVisitor withControlSemantics() { @@ -141,7 +133,8 @@ void addToken(HandlerData data, TokenType tokenType, Function data, TokenType tokenType, Function semantics, Function extractStartToken, Function extractEndToken) { + void addToken(HandlerData data, TokenType tokenType, Function semantics, Function extractStartToken, + Function extractEndToken) { data.collector().addToken(tokenType, semantics, data.entity(), extractStartToken, extractEndToken, data.variableRegistry()); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java index 22dd55f8f..d1f102ff3 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java @@ -1,11 +1,11 @@ package de.jplag.antlr; -import org.antlr.v4.runtime.ParserRuleContext; - import java.util.function.Function; +import org.antlr.v4.runtime.ParserRuleContext; + public class ContextDelegateVisitor extends DelegateVisitor { - private ContextVisitor contextVisitor; + private final ContextVisitor contextVisitor; public ContextDelegateVisitor(ContextVisitor delegate, Function mapper) { super(delegate, mapper); diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index dbc6e8bf1..2aa3a07a7 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -10,15 +10,14 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.TerminalNode; import de.jplag.TokenType; import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableRegistry; -import org.antlr.v4.runtime.tree.TerminalNode; /** * The visitor for nodes, or contexts. - * * @param The antlr type of the node. */ public class ContextVisitor extends AbstractVisitor { @@ -38,7 +37,6 @@ public class ContextVisitor extends AbstractVisitor /** * Add an action the visitor runs upon exiting the entity. - * * @param handler The action, takes the entity and the variable registry as parameter. * @return Self */ @@ -49,7 +47,6 @@ public AbstractVisitor onExit(BiConsumer handler) { /** * Add an action the visitor runs upon exiting the entity. - * * @param handler The action, takes the entity as parameter. * @return Self */ @@ -60,7 +57,6 @@ public AbstractVisitor onExit(Consumer handler) { /** * Tell the visitor that it should generate a token upon exiting the entity. Should only be invoked once per visitor. - * * @param tokenType The type of the token. * @return Self */ @@ -70,8 +66,8 @@ public ContextVisitor mapExit(TokenType tokenType) { } /** - * Behaves like mapEnter, but the created token will range from the beginning of this context to the end instead of only marking the beginning. - * + * Behaves like mapEnter, but the created token will range from the beginning of this context to the end instead of only + * marking the beginning. * @param tokenType The type of token to crate * @return Self */ @@ -94,10 +90,8 @@ public TerminalVisitor delegateTerminal(Function mapper) { /** * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token. - * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. - * - * Visits the terminal upon exiting this context - * + * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. Visits + * the terminal upon exiting this context * @param mapper The mapper function */ public TerminalVisitor delegateTerminalExit(Function mapper) { @@ -121,9 +115,8 @@ public ContextVisitor delegateContext(Function< /** * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be * invoked once per visitor. - * * @param enterTokenType The type of the token generated on enter. - * @param exitTokenType The type of the token generated on exit. + * @param exitTokenType The type of the token generated on exit. * @return Self */ public ContextVisitor mapEnterExit(TokenType enterTokenType, TokenType exitTokenType) { @@ -135,9 +128,8 @@ public ContextVisitor mapEnterExit(TokenType enterTokenType, TokenType exitTo /** * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be * invoked once per visitor. Alias for {@link #mapEnterExit(TokenType, TokenType)}. - * * @param enterTokenType The type of the token generated on enter. - * @param exitTokenType The type of the token generated on exit. + * @param exitTokenType The type of the token generated on exit. * @return Self */ public ContextVisitor map(TokenType enterTokenType, TokenType exitTokenType) { @@ -161,7 +153,6 @@ public ContextVisitor withSemantics(Supplier semantics) { /** * Tell the visitor that if it generates a token upon entering the entity, it should have semantics of type loop begin, * same for the exit and loop end. - * * @return Self */ public ContextVisitor withLoopSemantics() { @@ -172,7 +163,6 @@ public ContextVisitor withLoopSemantics() { /** * Tell the visitor that the entity represents a local scope. - * * @return Self */ public ContextVisitor addLocalScope() { @@ -183,7 +173,6 @@ public ContextVisitor addLocalScope() { /** * Tell the visitor that the entity represents a class scope. - * * @return Self */ public ContextVisitor addClassScope() { @@ -196,7 +185,7 @@ public ContextVisitor addClassScope() { * Exit a given entity, injecting the needed dependencies. */ void exit(HandlerData data) { - if(this.delegate != null) { + if (this.delegate != null) { this.delegate.delegateExit(data); return; } @@ -207,7 +196,7 @@ void exit(HandlerData data) { @Override void enter(HandlerData data) { - if(this.delegate != null) { + if (this.delegate != null) { this.delegate.delegateEnter(data); return; } @@ -226,7 +215,7 @@ Token extractEnterToken(T entity) { @Override boolean matches(T entity) { - if(this.delegate != null && !this.delegate.isPresent(entity)) { + if (this.delegate != null && !this.delegate.isPresent(entity)) { return false; } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java index bea70d851..1e4d679cb 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java @@ -32,7 +32,7 @@ public void delegateExit(HandlerData parentData) { public boolean isPresent(T entity) { try { return this.mapper.apply(entity) != null; - } catch (Exception e) { //If something goes wrong during mapping, the delegate is not present + } catch (Exception e) { // If something goes wrong during mapping, the delegate is not present return false; } } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java index 7e96e8cdd..9ebced353 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java @@ -1,9 +1,9 @@ package de.jplag.antlr; -import de.jplag.semantics.VariableRegistry; - import java.util.function.Function; +import de.jplag.semantics.VariableRegistry; + /** * Holds the data passed to the (quasi-static) listeners. */ diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java index c1d7e1317..436892aff 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java @@ -40,7 +40,8 @@ List getTokens() { } void addToken(TokenType jplagType, Function semanticsSupplier, T entity, - Function extractStartToken, Function extractEndToken, VariableRegistry variableRegistry) { + Function extractStartToken, Function extractEndToken, + VariableRegistry variableRegistry) { if (jplagType == null) { return; } diff --git a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java index 440bd2d37..e14926b43 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java @@ -1,8 +1,9 @@ package de.jplag.golang; -import de.jplag.antlr.AbstractAntlrLanguage; import org.kohsuke.MetaInfServices; +import de.jplag.antlr.AbstractAntlrLanguage; + @MetaInfServices(de.jplag.Language.class) public class GoLanguage extends AbstractAntlrLanguage { private static final String NAME = "Go Parser"; diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java index 2e9770868..43f0fd258 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -1,13 +1,129 @@ package de.jplag.golang; -import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.ContextVisitor; -import de.jplag.golang.grammar.GoParser.*; -import org.antlr.v4.runtime.ParserRuleContext; +import static de.jplag.golang.GoTokenType.ARGUMENT; +import static de.jplag.golang.GoTokenType.ARRAY_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.ARRAY_BODY_END; +import static de.jplag.golang.GoTokenType.ARRAY_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.ARRAY_ELEMENT; +import static de.jplag.golang.GoTokenType.ASSIGNMENT; +import static de.jplag.golang.GoTokenType.BREAK; +import static de.jplag.golang.GoTokenType.CASE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.CASE_BLOCK_END; +import static de.jplag.golang.GoTokenType.CONTINUE; +import static de.jplag.golang.GoTokenType.DEFER; +import static de.jplag.golang.GoTokenType.ELSE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.ELSE_BLOCK_END; +import static de.jplag.golang.GoTokenType.FALLTHROUGH; +import static de.jplag.golang.GoTokenType.FOR_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.FOR_BLOCK_END; +import static de.jplag.golang.GoTokenType.FOR_STATEMENT; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_END; +import static de.jplag.golang.GoTokenType.FUNCTION_DECLARATION; +import static de.jplag.golang.GoTokenType.FUNCTION_LITERAL; +import static de.jplag.golang.GoTokenType.FUNCTION_PARAMETER; +import static de.jplag.golang.GoTokenType.GO; +import static de.jplag.golang.GoTokenType.GOTO; +import static de.jplag.golang.GoTokenType.IF_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.IF_BLOCK_END; +import static de.jplag.golang.GoTokenType.IF_STATEMENT; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_BEGIN; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_END; +import static de.jplag.golang.GoTokenType.IMPORT_DECLARATION; +import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_END; +import static de.jplag.golang.GoTokenType.INTERFACE_DECLARATION; +import static de.jplag.golang.GoTokenType.INTERFACE_METHOD; +import static de.jplag.golang.GoTokenType.INVOCATION; +import static de.jplag.golang.GoTokenType.MAP_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.MAP_BODY_END; +import static de.jplag.golang.GoTokenType.MAP_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.MAP_ELEMENT; +import static de.jplag.golang.GoTokenType.MEMBER_DECLARATION; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_END; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_ELEMENT; +import static de.jplag.golang.GoTokenType.PACKAGE; +import static de.jplag.golang.GoTokenType.RECEIVER; +import static de.jplag.golang.GoTokenType.RECEIVE_STATEMENT; +import static de.jplag.golang.GoTokenType.RETURN; +import static de.jplag.golang.GoTokenType.SELECT_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.SELECT_BLOCK_END; +import static de.jplag.golang.GoTokenType.SELECT_STATEMENT; +import static de.jplag.golang.GoTokenType.SEND_STATEMENT; +import static de.jplag.golang.GoTokenType.SLICE_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.SLICE_BODY_END; +import static de.jplag.golang.GoTokenType.SLICE_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.SLICE_ELEMENT; +import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_END; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_END; +import static de.jplag.golang.GoTokenType.STRUCT_DECLARATION; +import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_END; +import static de.jplag.golang.GoTokenType.SWITCH_CASE; +import static de.jplag.golang.GoTokenType.SWITCH_STATEMENT; +import static de.jplag.golang.GoTokenType.TYPE_ASSERTION; +import static de.jplag.golang.GoTokenType.TYPE_CONSTRAINT; +import static de.jplag.golang.GoTokenType.VARIABLE_DECLARATION; import java.util.function.Function; -import static de.jplag.golang.GoTokenType.*; +import de.jplag.golang.grammar.GoParser.ArgumentsContext; +import de.jplag.golang.grammar.GoParser.ArrayTypeContext; +import de.jplag.golang.grammar.GoParser.AssignmentContext; +import de.jplag.golang.grammar.GoParser.BlockContext; +import de.jplag.golang.grammar.GoParser.BreakStmtContext; +import de.jplag.golang.grammar.GoParser.CommCaseContext; +import de.jplag.golang.grammar.GoParser.CommClauseContext; +import de.jplag.golang.grammar.GoParser.CompositeLitContext; +import de.jplag.golang.grammar.GoParser.ConstSpecContext; +import de.jplag.golang.grammar.GoParser.ContinueStmtContext; +import de.jplag.golang.grammar.GoParser.DeferStmtContext; +import de.jplag.golang.grammar.GoParser.ExprCaseClauseContext; +import de.jplag.golang.grammar.GoParser.ExprSwitchStmtContext; +import de.jplag.golang.grammar.GoParser.ExpressionContext; +import de.jplag.golang.grammar.GoParser.FallthroughStmtContext; +import de.jplag.golang.grammar.GoParser.FieldDeclContext; +import de.jplag.golang.grammar.GoParser.ForStmtContext; +import de.jplag.golang.grammar.GoParser.FunctionDeclContext; +import de.jplag.golang.grammar.GoParser.FunctionLitContext; +import de.jplag.golang.grammar.GoParser.GoStmtContext; +import de.jplag.golang.grammar.GoParser.GotoStmtContext; +import de.jplag.golang.grammar.GoParser.IfStmtContext; +import de.jplag.golang.grammar.GoParser.ImportDeclContext; +import de.jplag.golang.grammar.GoParser.ImportSpecContext; +import de.jplag.golang.grammar.GoParser.InterfaceTypeContext; +import de.jplag.golang.grammar.GoParser.KeyedElementContext; +import de.jplag.golang.grammar.GoParser.LiteralTypeContext; +import de.jplag.golang.grammar.GoParser.MapTypeContext; +import de.jplag.golang.grammar.GoParser.MethodDeclContext; +import de.jplag.golang.grammar.GoParser.MethodSpecContext; +import de.jplag.golang.grammar.GoParser.PackageClauseContext; +import de.jplag.golang.grammar.GoParser.ParameterDeclContext; +import de.jplag.golang.grammar.GoParser.ReceiverContext; +import de.jplag.golang.grammar.GoParser.RecvStmtContext; +import de.jplag.golang.grammar.GoParser.ReturnStmtContext; +import de.jplag.golang.grammar.GoParser.SelectStmtContext; +import de.jplag.golang.grammar.GoParser.SendStmtContext; +import de.jplag.golang.grammar.GoParser.ShortVarDeclContext; +import de.jplag.golang.grammar.GoParser.SliceTypeContext; +import de.jplag.golang.grammar.GoParser.StatementContext; +import de.jplag.golang.grammar.GoParser.StatementListContext; +import de.jplag.golang.grammar.GoParser.StructTypeContext; +import de.jplag.golang.grammar.GoParser.SwitchStmtContext; +import de.jplag.golang.grammar.GoParser.TypeAssertionContext; +import de.jplag.golang.grammar.GoParser.TypeCaseClauseContext; +import de.jplag.golang.grammar.GoParser.TypeNameContext; +import de.jplag.golang.grammar.GoParser.TypeSwitchStmtContext; +import de.jplag.golang.grammar.GoParser.VarDeclContext; +import org.antlr.v4.runtime.ParserRuleContext; + +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.ContextVisitor; public class GoListener extends AbstractAntlrListener { public GoListener() { @@ -64,9 +180,11 @@ private void functionDeclarations() { private void controlFlowRules() { visit(IfStmtContext.class).delegateTerminal(IfStmtContext::IF).map(IF_STATEMENT); - visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext)context.parent).block(0))).map(IF_BLOCK_BEGIN, IF_BLOCK_END); - //TODO no else token? - visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext)context.parent).block(1))).map(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END); + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext) context.parent).block(0))) + .map(IF_BLOCK_BEGIN, IF_BLOCK_END); + // TODO no else token? + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext) context.parent).block(1))) + .map(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END); visit(ForStmtContext.class).map(FOR_STATEMENT); visit(ForStmtContext.class).delegateTerminal(context -> context.block().L_CURLY()).map(FOR_BLOCK_BEGIN); @@ -150,18 +268,19 @@ private de.jplag.antlr.ContextVisitor visitComp } private ContextVisitor visitCompositeLitDelegate(Class type) { - return visit(CompositeLitContext.class, context -> context.literalType().children.stream().anyMatch(it -> type.isAssignableFrom(it.getClass()))); + return visit(CompositeLitContext.class, + context -> context.literalType().children.stream().anyMatch(it -> type.isAssignableFrom(it.getClass()))); } private ContextVisitor visitKeyedElement(Function typeGetter) { return visit(KeyedElementContext.class, context -> { CompositeLitContext parent = getAncestor(context, CompositeLitContext.class); - if(parent == null) { + if (parent == null) { return false; } LiteralTypeContext typeContext = parent.literalType(); - if(typeContext == null) { + if (typeContext == null) { return false; } diff --git a/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java b/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java index 0f6a76621..3e0a023fe 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java @@ -1,14 +1,15 @@ package de.jplag.golang; -import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.AbstractAntlrParserAdapter; -import de.jplag.golang.grammar.GoLexer; -import de.jplag.golang.grammar.GoParser; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.ParserRuleContext; +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.AbstractAntlrParserAdapter; +import de.jplag.golang.grammar.GoLexer; +import de.jplag.golang.grammar.GoParser; + public class GoParserAdapter extends AbstractAntlrParserAdapter { @Override protected Lexer createLexer(CharStream input) { From 98b4c677f6179ba920c2787e8107421144d9f6d1 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 9 Apr 2024 15:24:24 +0200 Subject: [PATCH 5/7] Spotless. --- .../golang/src/main/java/de/jplag/golang/GoListener.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java index 43f0fd258..cee4693e0 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -72,6 +72,10 @@ import java.util.function.Function; +import org.antlr.v4.runtime.ParserRuleContext; + +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.ContextVisitor; import de.jplag.golang.grammar.GoParser.ArgumentsContext; import de.jplag.golang.grammar.GoParser.ArrayTypeContext; import de.jplag.golang.grammar.GoParser.AssignmentContext; @@ -120,10 +124,6 @@ import de.jplag.golang.grammar.GoParser.TypeNameContext; import de.jplag.golang.grammar.GoParser.TypeSwitchStmtContext; import de.jplag.golang.grammar.GoParser.VarDeclContext; -import org.antlr.v4.runtime.ParserRuleContext; - -import de.jplag.antlr.AbstractAntlrListener; -import de.jplag.antlr.ContextVisitor; public class GoListener extends AbstractAntlrListener { public GoListener() { From 9c43b927ee269f4fe10267fc7763c352a666a71b Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 16 Apr 2024 12:35:47 +0200 Subject: [PATCH 6/7] Fixed sonarcloud issues for the go module --- .../main/java/de/jplag/antlr/ContextVisitor.java | 14 +++++++------- .../src/main/java/de/jplag/golang/GoListener.java | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 2aa3a07a7..e4dda7dc8 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -83,9 +83,9 @@ public AbstractVisitor mapRange(TokenType tokenType) { * @param mapper The mapper function */ public TerminalVisitor delegateTerminal(Function mapper) { - TerminalVisitor delegate = new TerminalVisitor((ignore) -> true); - this.delegate = new DelegateVisitor<>(delegate, parentData -> mapper.apply(parentData).getSymbol()); - return delegate; + TerminalVisitor delegateVisitor = new TerminalVisitor(ignore -> true); + this.delegate = new DelegateVisitor<>(delegateVisitor, parentData -> mapper.apply(parentData).getSymbol()); + return delegateVisitor; } /** @@ -95,10 +95,10 @@ public TerminalVisitor delegateTerminal(Function mapper) { * @param mapper The mapper function */ public TerminalVisitor delegateTerminalExit(Function mapper) { - TerminalVisitor delegate = new TerminalVisitor((ignore) -> true); - this.delegate = new DelegateVisitor<>(delegate, parentData -> mapper.apply(parentData).getSymbol()); + TerminalVisitor delegateVisitor = new TerminalVisitor(ignore -> true); + this.delegate = new DelegateVisitor<>(delegateVisitor, parentData -> mapper.apply(parentData).getSymbol()); this.delegate.mapOnExit(); - return delegate; + return delegateVisitor; } /** @@ -107,7 +107,7 @@ public TerminalVisitor delegateTerminalExit(Function mapper) { * @param mapper The mapper function */ public ContextVisitor delegateContext(Function mapper) { - ContextVisitor visitor = new ContextVisitor<>((ignore) -> true); + ContextVisitor visitor = new ContextVisitor<>(ignore -> true); this.delegate = new ContextDelegateVisitor<>(visitor, mapper); return visitor; } diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java index cee4693e0..2bc8c1e61 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -180,10 +180,9 @@ private void functionDeclarations() { private void controlFlowRules() { visit(IfStmtContext.class).delegateTerminal(IfStmtContext::IF).map(IF_STATEMENT); - visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext) context.parent).block(0))) - .map(IF_BLOCK_BEGIN, IF_BLOCK_END); - // TODO no else token? - visit(BlockContext.class, context -> context.parent instanceof IfStmtContext && context.equals(((IfStmtContext) context.parent).block(1))) + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext ifStmt && context.equals((ifStmt).block(0))).map(IF_BLOCK_BEGIN, + IF_BLOCK_END); + visit(BlockContext.class, context -> context.parent instanceof IfStmtContext ifStmt && context.equals((ifStmt).block(1))) .map(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END); visit(ForStmtContext.class).map(FOR_STATEMENT); From f89a93155659c78b0d380bec397d2eb78f6663c1 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 23 Apr 2024 09:47:14 +0200 Subject: [PATCH 7/7] Added javadoc --- .../jplag/antlr/ContextDelegateVisitor.java | 10 +++++++ .../java/de/jplag/antlr/DelegateVisitor.java | 26 +++++++++++++++++++ .../main/java/de/jplag/golang/GoListener.java | 4 +++ 3 files changed, 40 insertions(+) diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java index d1f102ff3..6c8bf10ae 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java @@ -4,9 +4,19 @@ import org.antlr.v4.runtime.ParserRuleContext; +/** + * Delegates visiting a {@link ParserRuleContext} to a different {@link ContextVisitor} derived by the given mapper + * function + * @param The original antlr type visited + * @param The target {@link ParserRuleContext} to visit instead + */ public class ContextDelegateVisitor extends DelegateVisitor { private final ContextVisitor contextVisitor; + /** + * @param delegate The visitor to delegate to + * @param mapper The mapper function used to derive the target antlr context + */ public ContextDelegateVisitor(ContextVisitor delegate, Function mapper) { super(delegate, mapper); this.contextVisitor = delegate; diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java index 1e4d679cb..465a7363f 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java @@ -2,33 +2,59 @@ import java.util.function.Function; +/** + * Delegates visiting of a given antlr entity to a visitor for a different antlr entity. + * @param The original antlr type visited + * @param The target antlr type + */ public class DelegateVisitor { private final AbstractVisitor delegate; protected final Function mapper; private boolean mapOnExit; + /** + * @param delegate The target visitor to use + * @param mapper The mapper function used to derive the target entity + */ public DelegateVisitor(AbstractVisitor delegate, Function mapper) { this.delegate = delegate; this.mapper = mapper; this.mapOnExit = false; } + /** + * Delegates entering the original context + * @param parentData The data of the original visitor + */ public void delegateEnter(HandlerData parentData) { if (!this.mapOnExit) { this.delegate.enter(parentData.derive(this.mapper)); } } + /** + * Makes this visitor map exit events to enter events. Used mostly for mapping exit events to terminal nodes, which only + * provide enter events + */ public void mapOnExit() { this.mapOnExit = true; } + /** + * Delegates exiting the original context + * @param parentData The data of the original visitor + */ public void delegateExit(HandlerData parentData) { if (this.mapOnExit) { this.delegate.enter(parentData.derive(this.mapper)); } } + /** + * Checks if the target entity is present in the given antlr entity + * @param entity The original antlr entity + * @return is present + */ public boolean isPresent(T entity) { try { return this.mapper.apply(entity) != null; diff --git a/languages/golang/src/main/java/de/jplag/golang/GoListener.java b/languages/golang/src/main/java/de/jplag/golang/GoListener.java index cee4693e0..e68ebde6b 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoListener.java @@ -125,6 +125,10 @@ import de.jplag.golang.grammar.GoParser.TypeSwitchStmtContext; import de.jplag.golang.grammar.GoParser.VarDeclContext; +/** + * Provides token extraction rules for {@link GoLanguage} Based on an older implementation of the language module; see + * JPlagGoListener.java in the history. + */ public class GoListener extends AbstractAntlrListener { public GoListener() { metaDeclarations();