Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve parsing of operator and function templates #732

Merged
merged 6 commits into from
Jan 1, 2024
Merged

Improve parsing of operator and function templates #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dfffed0
Improve parsing of function templates
HGuillemet Dec 24, 2023
2eb523d
Remove redundant infomap query
HGuillemet Dec 30, 2023
9623631
Replace Templates.SplitResult with plain list
HGuillemet Dec 30, 2023
514aeb9
Don't set context.virtualize for constructors.
HGuillemet Dec 30, 2023
95ff339
Instantiate templates with Info.define
HGuillemet Dec 31, 2023
7395919
Update CHANGELOG.md and fix nits
saudet Dec 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

* Improve `Parser` capabilities for `operator` and function templates ([pull #732](https://github.com/bytedeco/javacpp/pull/732))
* Fix `Parser` failing on nested initializer lists and on attributes found inside `enum` declarations
* Fix `Parser` for basic containers like `std::optional<std::pair<int,int> >` ([issue #718](https://github.com/bytedeco/javacpp/issues/718))
* Add support for `std::basic_string` basic container ([issue bytedeco/javacpp-presets#1311](https://github.com/bytedeco/javacpp-presets/issues/1311))
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/bytedeco/javacpp/tools/DeclarationList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,21 +66,21 @@ public boolean add(Declaration decl, String fullName) {
boolean add = true;
if (templateMap != null && templateMap.empty() && !decl.custom && (decl.type != null || decl.declarator != null)) {
// method templates cannot be declared in Java, but make sure to make their
// info available on request (when Info.javaNames is set) to be able to create instances
// info available on request (when Info.javaNames or Info.define is set) to be able to create instances
if (infoIterator == null) {
Type type = templateMap.type = decl.type;
Declarator dcl = templateMap.declarator = decl.declarator;
for (String name : new String[] {fullName, dcl != null ? dcl.cppName : type.cppName}) {
for (String name : new String[] {fullName, dcl != null ? (dcl.type.constructor ? Parser.constructorName(dcl.cppName) : dcl.cppName) : type.cppName}) {
if (name == null) {
continue;
}
List<Info> infoList = infoMap.get(name);
boolean hasJavaName = false;
boolean hasJavaNameOrDefine = false;
for (Info info : infoList) {
hasJavaName |= info.javaNames != null && info.javaNames.length > 0;
hasJavaNameOrDefine |= info.javaNames != null && info.javaNames.length > 0 || info.define;
}
if (!decl.function || hasJavaName) {
infoIterator = infoList.size() > 0 ? infoList.listIterator() : null;
if (!decl.function || hasJavaNameOrDefine) {
infoIterator = infoList.size() > 0 ? new ArrayList<>(infoList).listIterator() : null;
break;
}
}
Expand Down
53 changes: 15 additions & 38 deletions src/main/java/org/bytedeco/javacpp/tools/InfoMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ public class InfoMap extends HashMap<String,List<Info>> {
"std::function", "std::basic_string"))
.put(new Info("basic/types").cppTypes("signed", "unsigned", "char", "short", "int", "long", "bool", "float", "double",
"__int8", "__int16", "__int32", "__int64", "_Bool", "_Complex", "_Imaginary", "complex", "imaginary"))
.put(new Info("deprecated").annotations("@Deprecated"))
.put(new Info("noexcept").annotations("@NoException(true)"))

.put(new Info("__COUNTER__").cppText("#define __COUNTER__ 0"))
Expand Down Expand Up @@ -185,6 +184,21 @@ String normalize(String name, boolean unconst, boolean untemplate) {
if (name == null || name.length() == 0 || name.startsWith("basic/")) {
return name;
}
if (untemplate) {
// Remove template arguments in the last NS component only, and not in parameters, if any
List<String> comps = Templates.splitNamespace(name, true);
int paramsIdx = comps.size() - 1;
String lastComp = comps.get(paramsIdx - 1);
comps.set(paramsIdx - 1, Templates.strip(lastComp));
name = comps.get(0);
for (int i = 1; i < paramsIdx; i++) {
name += "::" + comps.get(i);
}
name += comps.get(paramsIdx);
if (name.isEmpty()) {
return name;
}
}
boolean foundConst = false, simpleType = true;
String prefix = null;
Token[] tokens = new Tokenizer(name, null, 0).tokenize();
Expand Down Expand Up @@ -216,43 +230,6 @@ String normalize(String name, boolean unconst, boolean untemplate) {
for (int i = 1; i < n; i++) {
name += " " + tokens[i].value;
}
} else if (untemplate) {
int count = 0, lastColon = -1, template = -1, parameters = n;
for (int i = 0; i < n; i++) {
if (tokens[i].match('<')) {
count++;
} else if (tokens[i].match('>')) {
count--;
}
if (count == 0 && tokens[i].match("::")) {
lastColon = i;
} else if (count == 0 && tokens[i].match('(')) {
parameters = i;
break;
}
}
for (int i = lastColon + 1; i < parameters; i++) {
if (tokens[i].match('<')) {
if (count == 0) {
template = i;
}
count++;
} else if (tokens[i].match('>')) {
count--;
if (count == 0 && i + 1 != parameters) {
template = -1;
}
}
}
if (template >= 0) {
name = foundConst ? "const " : "";
for (int i = 0; i < template; i++) {
name += tokens[i];
}
for (int i = parameters; i < n; i++) {
name += tokens[i].spacing + tokens[i];
}
}
}
if (unconst && foundConst) {
name = name.substring(name.indexOf("const") + 5);
Expand Down
134 changes: 92 additions & 42 deletions src/main/java/org/bytedeco/javacpp/tools/Parser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,17 @@ static String upcastMethodName(String javaName) {
return "as" + Character.toUpperCase(shortName.charAt(0)) + shortName.substring(1);
}

/** Returns the name of the constructor of class cppName, to be used as keys in infoMap */
/**
* Constructors have 2 kinds of fully qualified name:
* the calling name of a constructor, used when calling the constructor, e.g.:
* NS::CN(int)
* and the declaration name, used when defining the constructor outside its class or when referencing a constructor
* with using to inherit constructors of base class.
* NS::CN::CN(int)
* Declarator.cppName contains the calling name, and this method returns the declaration name.
* Keys in info map should use the declaration name, because the calling name cannot specify
* arguments in case of constructor templates, and to avoid confusion between classes and constructores info.
*/
static String constructorName(String cppName) {
String constructorName = Templates.strip(cppName);
int namespace = constructorName.lastIndexOf("::");
Expand Down Expand Up @@ -752,6 +762,28 @@ Type[] templateArguments(Context context) throws ParserException {
return arguments.toArray(new Type[0]);
}

/**
* Read and return the operator following an operator keyword:
* any of new, delete, + - * / % ^ & | ~ ! = < > += -= *= /= %= ^= &= |= << >> >>= <<= == != <= >= <=>(since C++20) && || ++ -- , ->* -> ( ) [ ]
* taking care of template arguments, if any.
*/
private String operator(Context context) throws ParserException {
String res = tokens.get().toString(); // Can be '('
int lenFirstToken = res.length();
String s = "";
tokens.next();
int backIndex = tokens.index;
for (Token token = tokens.get(); !token.match('(', ';', Token.EOF); token = tokens.next()) {
s += token;
}
s = Templates.strip(s);
tokens.index = backIndex;
for (Token token = tokens.get(); s.length() > res.length() - lenFirstToken; token = tokens.next()) {
res += token;
}
return res;
}

Type type(Context context) throws ParserException {
return type(context, false);
}
Expand Down Expand Up @@ -865,7 +897,8 @@ Type type(Context context, boolean definition) throws ParserException {
} else if (type.cppName.endsWith("::")) {
type.operator = true;
tokens.next();
break;
type.cppName += operator(context);
continue;
} else {
break;
}
Expand Down Expand Up @@ -957,18 +990,20 @@ Type type(Context context, boolean definition) throws ParserException {

// perform template substitution
if (context.templateMap != null) {
List<String> types = Templates.splitNamespace(type.cppName);
List<String> types = Templates.splitNamespace(type.cppName, true);
String separator = "";
type.cppName = "";
List<Type> arguments = new ArrayList<>();
for (String t : types) {
Type t2 = context.templateMap.get(t);
type.cppName += separator + (t2 != null ? t2.cppName : t);
int paramsIdx = types.size() - 1;
for (int i = 0; i < paramsIdx; i++) {
Type t2 = context.templateMap.get(types.get(i));
type.cppName += separator + (t2 != null ? t2.cppName : types.get(i));
if (t2 != null && t2.arguments != null) {
arguments.addAll(Arrays.asList(t2.arguments));
}
separator = "::";
}
type.cppName += types.get(paramsIdx);
if (arguments.size() > 0) {
type.arguments = arguments.toArray(new Type[0]);
}
Expand Down Expand Up @@ -1109,22 +1144,15 @@ Type type(Context context, boolean definition) throws ParserException {
}
}
if (context.cppName != null && type.javaName.length() > 0) {
String cppName = type.cppName;
String groupName = context.cppName;
String cppNameStripped = Templates.strip(cppName);
String groupNameStripped = Templates.strip(groupName);
if (cppNameStripped.length() == cppName.length() && groupNameStripped.length() != groupName.length()) {
groupName = groupNameStripped;
} else if (cppNameStripped.length() != cppName.length() && groupNameStripped.length() == groupName.length()) {
cppName = cppNameStripped;
}
String cppName = Templates.strip(type.cppName);
String groupName = Templates.strip(context.cppName);
List<String> cppNameSplit = Templates.splitNamespace(cppName);
List<String> groupNameSplit = Templates.splitNamespace(groupName);
if (cppNameSplit.size() == 1 && groupNameSplit.size() > 1)
groupName = groupNameSplit.get(groupNameSplit.size() - 1);
else if (cppNameSplit.size() > 1 && groupNameSplit.size() == 1)
cppName = cppNameSplit.get(cppNameSplit.size() - 1);
if (cppName.equals(groupName) || groupName.startsWith(cppName + "<")) {
if (cppName.equals(groupName)) {
type.constructor = !type.destructor && !type.operator
&& type.indirections == 0 && !type.reference && tokens.get().match('(', ':');
}
Expand Down Expand Up @@ -1347,11 +1375,8 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole
} else if (token.match(Token.OPERATOR)) {
dcl.operator = true;
if (!tokens.get(1).match(Token.IDENTIFIER) || tokens.get(1).match(Token.NEW, Token.DELETE)) {
// assume we can have any symbols until the first open parenthesis
dcl.cppName += "operator " + tokens.next();
for (token = tokens.next(); !token.match(Token.EOF, '('); token = tokens.next()) {
dcl.cppName += token;
}
tokens.next();
dcl.cppName += "operator " + operator(context);
break;
}
} else if (token.match('<')) {
Expand Down Expand Up @@ -2130,14 +2155,16 @@ Parameters parameters(Context context, int infoNumber, boolean useDefaults) thro
// perform template substitution
String cppName = token.value;
if (context.templateMap != null) {
List<String> types = Templates.splitNamespace(cppName);
List<String> types = Templates.splitNamespace(cppName, true);
String separator = "";
cppName = "";
for (String t : types) {
Type t2 = context.templateMap.get(t);
cppName += separator + (t2 != null ? t2.cppName : t);
int paramsIdx = types.size() - 1;
for (int i = 0; i < paramsIdx; i++) {
Type t2 = context.templateMap.get(types.get(i));
cppName += separator + (t2 != null ? t2.cppName : types.get(i));
separator = "::";
}
cppName += types.get(paramsIdx);
}

// try to qualify all the identifiers
Expand Down Expand Up @@ -2343,14 +2370,16 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
}

boolean isQualified = Templates.splitNamespace(dcl.cppName).size() > 1;
if (context.namespace != null && !isQualified) {
if (context.namespace != null && !isQualified && !(type.constructor || type.destructor)) {
dcl.cppName = context.namespace + "::" + dcl.cppName;
}
Info info = null, fullInfo = null;
String fullname = dcl.cppName, fullname2 = dcl.cppName;
String templateArgs = declList.templateMap != null ? declList.templateMap.toString() : "";
String fullname = dcl.cppName + templateArgs;
String param1 = "", param2 = "";
if (dcl.parameters != null) {
fullname += "(";
fullname2 += "(";
param1 = "(";
param2 = "(";
String separator = "";
for (Declarator d : dcl.parameters.declarators) {
if (d != null) {
Expand All @@ -2376,25 +2405,42 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
if (d.type.constPointer && !s.endsWith(" const")) {
s = s + " const";
}
fullname += separator + s;
fullname2 += separator + s2;
param1 += separator + s;
param2 += separator + s2;
separator = ", ";
}
}
info = fullInfo = infoMap.getFirst(fullname += ")", false);
param1 += ")";
param2 += ")";
fullname += param1;
info = fullInfo = infoMap.getFirst(fullname, false);
if (info == null) {
info = infoMap.getFirst(fullname2 += ")", false);
info = infoMap.getFirst(dcl.cppName + templateArgs + param2, false);
if (info == null && !templateArgs.isEmpty()) {
info = infoMap.getFirst(dcl.cppName + param1, false);
if (info == null) {
info = infoMap.getFirst(dcl.cppName + param2, false);
}
}
}
}
if (info == null) {
if (type.constructor) {
// get Info explicitly associated with all constructors
List<String> cppNameSplit = Templates.splitNamespace(dcl.cppName);
String name = Templates.strip(cppNameSplit.get(cppNameSplit.size() - 1));
info = fullInfo = infoMap.getFirst(dcl.cppName + "::" + name);
String name = constructorName(dcl.cppName);
fullname = name + templateArgs + param1;
info = fullInfo = infoMap.getFirst(fullname);
if (info == null) {
info = fullInfo = infoMap.getFirst(name + templateArgs + param2);
if (info == null) {
info = fullInfo = infoMap.getFirst(name + templateArgs);
}
}
}
// For constructor, we'd better not make this lookup, because of confusion
// with the class info. Kept for now for backwards compatibility.
if (info == null) {
info = infoMap.getFirst(dcl.cppName);
info = infoMap.getFirst(dcl.cppName + templateArgs);
}
if (!type.constructor && !type.destructor && !type.operator && (context.templateMap == null || context.templateMap.full())) {
infoMap.put(info != null ? new Info(info).cppNames(fullname).javaNames(null) : new Info(fullname));
Expand Down Expand Up @@ -2466,7 +2512,12 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti

type = functionAfter(context, decl, dcl, type);
context = new Context(context);
context.virtualize = (context.virtualize && type.virtual) || (info != null && info.virtualize);

// Virtualize the function if class is virtualized and C++ function is virtual
// or if function is explicitly virtualized with info.
// Exclude constructor case since we may have looked up the info of the class in lieu of
// the info of the constructor, and constructors cannot be virtualized.
context.virtualize = (context.virtualize && type.virtual) || (info != null && info.virtualize && !type.constructor);

List<Declarator> prevDcl = new ArrayList<Declarator>();
boolean first = true;
Expand Down Expand Up @@ -2526,8 +2577,7 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
}

// use Java names that we may get here but that declarator() did not catch
String parameters = fullname.substring(dcl.cppName.length());
for (String name : context.qualify(dcl.cppName, parameters)) {
for (String name : context.qualify(dcl.cppName, param1)) {
if ((infoMap.getFirst(name, false)) != null) {
dcl.cppName = name;
break;
Expand All @@ -2539,8 +2589,8 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
if (context.namespace != null && localName2.startsWith(context.namespace + "::")) {
localName2 = dcl.cppName.substring(context.namespace.length() + 2);
}
if (localName2.endsWith(parameters)) {
localName2 = localName2.substring(0, localName2.length() - parameters.length());
if (localName2.endsWith(param1)) {
localName2 = localName2.substring(0, localName2.length() - param1.length());
}
if (fullInfo != null && fullInfo.javaNames != null && fullInfo.javaNames.length > 0) {
dcl.javaName = fullInfo.javaNames[0];
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/org/bytedeco/javacpp/tools/TemplateMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
package org.bytedeco.javacpp.tools;

import java.util.LinkedHashMap;
import java.util.Map;

/**
*
Expand Down Expand Up @@ -67,4 +68,20 @@ Type get(String key) {
return value;
}
}

@Override
public String toString() {
String s = "<";
for (Map.Entry<String, Type> e : entrySet()) {
if (s.length() > 1) {
s += ",";
}
Type t = e.getValue();
s += t != null ? t.cppName : e.getKey();
}
if (s.endsWith(">")) {
s += " ";
}
return s + ">";
}
}
Loading