From 3f20b42cae46acf732f2056e439f13d056003579 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Fri, 3 Nov 2023 17:54:34 +0100 Subject: [PATCH 01/21] Don't interpret "= default" or "= delete" to mean this is an abstract method --- generator/parser/ast.h | 2 ++ generator/parser/binder.cpp | 3 ++- generator/parser/parser.cpp | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/generator/parser/ast.h b/generator/parser/ast.h index 06b9baacf..181c63883 100644 --- a/generator/parser/ast.h +++ b/generator/parser/ast.h @@ -512,6 +512,8 @@ struct InitializerAST: public AST InitializerClauseAST *initializer_clause{}; ExpressionAST *expression{}; + bool isDefault{}; + bool isDeleted{}; }; struct InitializerClauseAST: public AST diff --git a/generator/parser/binder.cpp b/generator/parser/binder.cpp index f8bbd3a76..13a424741 100644 --- a/generator/parser/binder.cpp +++ b/generator/parser/binder.cpp @@ -282,7 +282,8 @@ void Binder::declare_symbol(SimpleDeclarationAST *node, InitDeclaratorAST *init_ fun->setAccessPolicy(_M_current_access); fun->setFunctionType(_M_current_function_type); fun->setName(name_cc.name()); - fun->setAbstract(init_declarator->initializer != 0); + InitializerAST* initializer = init_declarator->initializer; + fun->setAbstract(initializer && !initializer->isDefault && !initializer->isDeleted); // must be "= 0" fun->setConstant(declarator->fun_cv != 0); fun->setException(exceptionSpecToString(declarator->exception_spec)); diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index 18a60cc57..35107396f 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -2305,8 +2305,19 @@ bool Parser::parseInitializer(InitializerAST *&node) if (tk == '=') { nextToken(); - - if (!parseInitializerClause(ast->initializer_clause)) + tk = token_stream.lookAhead(); + // this is also used for methods: + if (tk == Token_delete) + { + ast->isDeleted = true; + nextToken(); + } + else if (tk == Token_default) + { + ast->isDefault = true; + nextToken(); + } + else if (!parseInitializerClause(ast->initializer_clause)) { reportError(("Initializer clause expected")); } From 9a08b85a0e654b5287c7883f6bbb0d71cb456da7 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Mon, 6 Nov 2023 19:23:20 +0100 Subject: [PATCH 02/21] Deleted methods need to marked as "invalid" to be handled correctly --- generator/abstractmetabuilder.cpp | 5 +++++ generator/parser/binder.cpp | 1 + generator/parser/codemodel.cpp | 10 ++++++++++ generator/parser/codemodel.h | 4 ++++ 4 files changed, 20 insertions(+) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index 1475f0872..efc6a0b3a 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -1593,6 +1593,11 @@ AbstractMetaFunction *AbstractMetaBuilder::traverseFunction(FunctionModelItem fu meta_function->setFunctionType(AbstractMetaFunction::SlotFunction); } + if (function_item->isDeleted()) { + meta_function->setInvalid(true); + return meta_function; + } + ArgumentList arguments = function_item->arguments(); AbstractMetaArgumentList meta_arguments; diff --git a/generator/parser/binder.cpp b/generator/parser/binder.cpp index 13a424741..0684eaac0 100644 --- a/generator/parser/binder.cpp +++ b/generator/parser/binder.cpp @@ -283,6 +283,7 @@ void Binder::declare_symbol(SimpleDeclarationAST *node, InitDeclaratorAST *init_ fun->setFunctionType(_M_current_function_type); fun->setName(name_cc.name()); InitializerAST* initializer = init_declarator->initializer; + fun->setDeleted(initializer && initializer->isDeleted); fun->setAbstract(initializer && !initializer->isDefault && !initializer->isDeleted); // must be "= 0" fun->setConstant(declarator->fun_cv != 0); fun->setException(exceptionSpecToString(declarator->exception_spec)); diff --git a/generator/parser/codemodel.cpp b/generator/parser/codemodel.cpp index a32c0614d..d2b8523df 100644 --- a/generator/parser/codemodel.cpp +++ b/generator/parser/codemodel.cpp @@ -731,6 +731,16 @@ void _FunctionModelItem::setAbstract(bool isAbstract) _M.isAbstract = isAbstract; } +bool _FunctionModelItem::isDeleted() const +{ + return _M.isDeleted; +} + +void _FunctionModelItem::setDeleted(bool isDeleted) +{ + _M.isDeleted = isDeleted; +} + // Qt bool _FunctionModelItem::isInvokable() const { diff --git a/generator/parser/codemodel.h b/generator/parser/codemodel.h index efcee3d35..3ee5d125d 100644 --- a/generator/parser/codemodel.h +++ b/generator/parser/codemodel.h @@ -566,6 +566,9 @@ class _FunctionModelItem: public _MemberModelItem bool isAbstract() const; void setAbstract(bool isAbstract); + bool isDeleted() const; + void setDeleted(bool isDeleted); + bool isVariadics() const; void setVariadics(bool isVariadics); @@ -589,6 +592,7 @@ class _FunctionModelItem: public _MemberModelItem uint isVirtual: 1; uint isInline: 1; uint isAbstract: 1; + uint isDeleted: 1; uint isExplicit: 1; uint isVariadics: 1; uint isInvokable : 1; // Qt From 6abdb1f44278cd5ffbf9644e48cd4a4f2557b8b3 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Sat, 4 Nov 2023 14:47:57 +0100 Subject: [PATCH 03/21] Cover cases where noexcept comes with an expression --- generator/parser/parser.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index 35107396f..3e2a6afba 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -2178,6 +2178,13 @@ bool Parser::parseExceptionSpecification(ExceptionSpecificationAST *&node) { // ignore noexcept nextToken(); + if (token_stream.lookAhead() == '(') + { + nextToken(); + ExpressionAST* node; + parseCommaExpression(node); + CHECK(')'); + } return true; } From cc312c213c67d3e589568a889a431526ea9b48c0 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Fri, 3 Nov 2023 23:30:24 +0100 Subject: [PATCH 04/21] Implement and use Parser::skipFunctionBody this way we can avoid some parser errors. This is implemented by looking for matching braces. I hope there is no case where unmatched braces are allowed (strings and comments should be handled by the tokenizer). --- generator/abstractmetabuilder.cpp | 1 + generator/parser/parser.cpp | 76 ++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index efc6a0b3a..e36b03003 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -409,6 +409,7 @@ bool AbstractMetaBuilder::build() file.close(); Control control; + control.setSkipFunctionBody(true); Parser p(&control); pool __pool; diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index 3e2a6afba..f249c979a 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -50,6 +50,7 @@ #include #include +#include #define ADVANCE(tk, descr) \ { \ @@ -3420,13 +3421,76 @@ bool Parser::parseDeclarationInternal(DeclarationAST *&node) return false; } -bool Parser::skipFunctionBody(StatementAST *&) +void popStackUntil(std::vector& stack, int value) { -#if defined(__GNUC__) -#pragma GCC warning "Parser::skipFunctionBody() -- implement me" -#endif - Q_ASSERT(0); // ### not implemented - return 0; + while (!stack.empty() && stack.back() != value) + { + stack.pop_back(); + } + if (!stack.empty()) + { + stack.pop_back(); + } +} + +bool Parser::skipFunctionBody(StatementAST *& node) +{ + std::size_t start = token_stream.cursor(); + std::vector braceStack; + if (token_stream.lookAhead() != '{') + { + reportError("'{' expected (block start)"); + return false; + } + braceStack.push_back('{'); + nextToken(); + while (!braceStack.empty()) + { + int tk = token_stream.lookAhead(); + switch (tk) + { + // handle opening braces: + case '{': + case '[': + case '(': + braceStack.push_back(tk); + break; + // handle closing braces: + case '}': + if (braceStack.back() != '{') + { + reportError("misbalanced '}' brace"); + } + popStackUntil(braceStack, '{'); + break; + case ']': + if (braceStack.back() != '[') + { + reportError("misbalanced ']' brace"); + } + popStackUntil(braceStack, '['); + break; + case ')': + if (braceStack.back() != '(') + { + reportError("misbalanced ')' brace"); + } + popStackUntil(braceStack, '('); + break; + case Token_EOF: + reportError("unexpected EOF while skipping block"); + braceStack.clear(); + break; + default: + // skip everything else + ; + } + nextToken(); + } + CompoundStatementAST* ast = CreateNode(_M_pool); + UPDATE_POS(ast, start, token_stream.cursor()); + node = ast; + return true; } bool Parser::parseFunctionBody(StatementAST *&node) From e16110d39fd11aec77cc5784178d52c4f2f66871 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Thu, 9 Nov 2023 17:33:06 +0100 Subject: [PATCH 05/21] Simplify skipFunctionBody by only looking at curly braces --- generator/parser/parser.cpp | 50 +++++-------------------------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index f249c979a..d216e948c 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -3421,65 +3421,27 @@ bool Parser::parseDeclarationInternal(DeclarationAST *&node) return false; } -void popStackUntil(std::vector& stack, int value) -{ - while (!stack.empty() && stack.back() != value) - { - stack.pop_back(); - } - if (!stack.empty()) - { - stack.pop_back(); - } -} - bool Parser::skipFunctionBody(StatementAST *& node) { std::size_t start = token_stream.cursor(); - std::vector braceStack; - if (token_stream.lookAhead() != '{') - { - reportError("'{' expected (block start)"); - return false; - } - braceStack.push_back('{'); - nextToken(); - while (!braceStack.empty()) + ADVANCE('{', "{"); + int braceCount = 1; + while (braceCount) { int tk = token_stream.lookAhead(); switch (tk) { // handle opening braces: case '{': - case '[': - case '(': - braceStack.push_back(tk); + braceCount++; break; // handle closing braces: case '}': - if (braceStack.back() != '{') - { - reportError("misbalanced '}' brace"); - } - popStackUntil(braceStack, '{'); - break; - case ']': - if (braceStack.back() != '[') - { - reportError("misbalanced ']' brace"); - } - popStackUntil(braceStack, '['); - break; - case ')': - if (braceStack.back() != '(') - { - reportError("misbalanced ')' brace"); - } - popStackUntil(braceStack, '('); + braceCount--; break; case Token_EOF: reportError("unexpected EOF while skipping block"); - braceStack.clear(); + braceCount = 0; break; default: // skip everything else From 4de21d5e4d51ad6771db47ec9c76162e69a03c4f Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Fri, 3 Nov 2023 18:18:18 +0100 Subject: [PATCH 06/21] Avoid currently unsupported use of "noexcept(...)" --- generator/qtscript_masterinclude.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/generator/qtscript_masterinclude.h b/generator/qtscript_masterinclude.h index 52d63c244..29a6f8a3b 100644 --- a/generator/qtscript_masterinclude.h +++ b/generator/qtscript_masterinclude.h @@ -73,6 +73,10 @@ # undef Q_CLANG_QDOC #endif +// it seems this can be safely ignored (otherwise generator currently stumbles over use of noexcept): +#define Q_DECLARE_SHARED(TYPE) +#define Q_DECLARE_SHARED_NOT_MOVABLE_UNTIL_QT6(TYPE) + #include #include #include From a9cf7ac1ccaae479561e6fcc3d19585138756ea1 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Sat, 4 Nov 2023 16:24:00 +0100 Subject: [PATCH 07/21] Ignore some more macros that lead to parser errors --- generator/qtscript_masterinclude.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/generator/qtscript_masterinclude.h b/generator/qtscript_masterinclude.h index 29a6f8a3b..10f9319b6 100644 --- a/generator/qtscript_masterinclude.h +++ b/generator/qtscript_masterinclude.h @@ -77,6 +77,23 @@ #define Q_DECLARE_SHARED(TYPE) #define Q_DECLARE_SHARED_NOT_MOVABLE_UNTIL_QT6(TYPE) +// also ignore deprecation macros for now: +#define QT_DEPRECATED_VERSION_X_5(minor, text) +#define QT_DEPRECATED_VERSION_X(major, minor, text) + +#define QT_DEPRECATED_VERSION_5(minor) +#define QT_DEPRECATED_VERSION(major, minor) + +// we also don't use this: +#define Q_CLASSINFO(name, value) +#define Q_PRIVATE_PROPERTY(d, text) + +// treat QDOC_PROPERTY like Q_PROPERTY +#define QDOC_PROPERTY(text) Q_PROPERTY(text) + +// don't need this: +#define Q_REVISION(v) + #include #include #include From 09caf99174eb84b0bfb609f4bc04b033191999e8 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Sat, 4 Nov 2023 16:31:27 +0100 Subject: [PATCH 08/21] Support basic new-style initializers --- generator/parser/parser.cpp | 36 +++++++++++++++++++++++++++++------- generator/parser/parser.h | 2 +- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index d216e948c..f2b6f5f35 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -2257,7 +2257,7 @@ bool Parser::parseInitDeclarator(InitDeclaratorAST *&node) } InitializerAST *init = 0; - parseInitializer(init); + parseInitializer(init, !decl->parameter_declaration_clause); InitDeclaratorAST *ast = CreateNode(_M_pool); ast->declarator = decl; @@ -2300,12 +2300,12 @@ bool Parser::parseBaseClause(BaseClauseAST *&node) return true; } -bool Parser::parseInitializer(InitializerAST *&node) +bool Parser::parseInitializer(InitializerAST *&node, bool allowNewStyle) { std::size_t start = token_stream.cursor(); int tk = token_stream.lookAhead(); - if (tk != '=' && tk != '(') + if (tk != '=' && tk != '(' && !(allowNewStyle && tk == '{')) return false; InitializerAST *ast = CreateNode(_M_pool); @@ -2336,6 +2336,15 @@ bool Parser::parseInitializer(InitializerAST *&node) parseCommaExpression(ast->expression); CHECK(')'); } + else if (tk == '{') + { + nextToken(); + if (token_stream.lookAhead() != '}') { + parseCommaExpression(ast->expression); + } + // new-style initializers may be empty (default value) + CHECK('}'); + } UPDATE_POS(ast, start, token_stream.cursor()); node = ast; @@ -2376,10 +2385,23 @@ bool Parser::parseMemInitializer(MemInitializerAST *&node) return false; } - ADVANCE('(', "("); - ExpressionAST *expr = 0; - parseCommaExpression(expr); - ADVANCE(')', ")"); + ExpressionAST* expr = 0; + if (token_stream.lookAhead() == '{') + { + // new style initializers + ADVANCE('{', "{"); + if (token_stream.lookAhead() != '}') { + parseCommaExpression(expr); + } + // new-style initializers may be empty (default value) + ADVANCE('}', "}"); + } + else + { + ADVANCE('(', "("); + parseCommaExpression(expr); + ADVANCE(')', ")"); + } MemInitializerAST *ast = CreateNode(_M_pool); ast->initializer_id = initId; diff --git a/generator/parser/parser.h b/generator/parser/parser.h index 2f3185849..36d77c57c 100644 --- a/generator/parser/parser.h +++ b/generator/parser/parser.h @@ -116,7 +116,7 @@ class Parser bool templArgs = false); bool parseInitDeclarator(InitDeclaratorAST *&node); bool parseInitDeclaratorList(const ListNode *&node); - bool parseInitializer(InitializerAST *&node); + bool parseInitializer(InitializerAST *&node, bool allowNewStyle); bool parseInitializerClause(InitializerClauseAST *&node); bool parseLabeledStatement(StatementAST *&node); bool parseLinkageBody(LinkageBodyAST *&node); From 66df824cc1fabb9ee117c8be873706cc310d7c4c Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Mon, 6 Nov 2023 19:22:25 +0100 Subject: [PATCH 09/21] Add missing enum used as default value for a method --- generator/typesystem_network.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/typesystem_network.xml b/generator/typesystem_network.xml index f626f1393..abb862578 100644 --- a/generator/typesystem_network.xml +++ b/generator/typesystem_network.xml @@ -52,6 +52,7 @@ + From 37f98c7383953159df73538d625639994dd417cc Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Mon, 6 Nov 2023 19:24:03 +0100 Subject: [PATCH 10/21] Automatically mark methods with r-value references (&&) as invalid --- generator/abstractmetabuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index e36b03003..161027e82 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -1702,7 +1702,7 @@ AbstractMetaType *AbstractMetaBuilder::translateType(const TypeInfo &_typei, boo } - if (typei.isFunctionPointer()) { + if (typei.isFunctionPointer() || typei.isRvalueReference()) { // function pointers or r-value references are not supported *ok = false; return 0; } From f3c0c9ab8eb71afb45e5af53472ed87cf50edbaf Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Wed, 8 Nov 2023 13:27:25 +0100 Subject: [PATCH 11/21] Don't create wrappers for cbegin and cend --- generator/typesystem_core.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/typesystem_core.xml b/generator/typesystem_core.xml index 360401bbb..26b36663b 100644 --- a/generator/typesystem_core.xml +++ b/generator/typesystem_core.xml @@ -574,6 +574,8 @@ + + From d9901c589e3749e4e4a179bfeacd790b7e843cea Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Wed, 8 Nov 2023 13:37:09 +0100 Subject: [PATCH 12/21] Hopefully make generator immune against iterator order changes for typeMap Previously the includes and generated methods for e.g. QItemSelection would change depending on the order in which the AbstractMetaClass entries were generated, which in turn would change which definition of QList would be seen first. I now try to mark the "main" definition containing the actual methods and such. For Qt 5.15 this seems to work at least, but I admit that I still don't understand the generator completely. --- generator/abstractmetabuilder.cpp | 24 ++++++++++++++---------- generator/abstractmetabuilder.h | 2 +- generator/abstractmetalang.h | 7 ++++++- generator/parser/binder.cpp | 1 + generator/parser/codemodel.h | 6 ++++++ generator/typeparser.h | 2 +- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index 161027e82..cfae4fa27 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -617,7 +617,15 @@ void AbstractMetaBuilder::addAbstractMetaClass(AbstractMetaClass *cls) cls->setOriginalAttributes(cls->attributes()); if (cls->typeEntry()->isContainer()) { - m_templates << cls; + QString name = cls->typeEntry()->name(); + if (cls->functions().size() || cls->baseClassNames().size()) { + if (!m_templates.contains(name)) { + m_templates[name] = cls; + } + else { + ReportHandler::warning(QString("Duplicate container type template '%1'").arg(name)); + } + } } else { m_meta_classes << cls; if (cls->typeEntry()->designatedInterface()) { @@ -1096,6 +1104,7 @@ AbstractMetaClass *AbstractMetaBuilder::traverseClass(ClassModelItem class_item) template_args.append(param_type); } meta_class->setTemplateArguments(template_args); + meta_class->setHasActualDeclaration(class_item->hasActualDeclaration()); parseQ_Property(meta_class, class_item->propertyDeclarations()); @@ -1130,8 +1139,9 @@ AbstractMetaClass *AbstractMetaBuilder::traverseClass(ClassModelItem class_item) m_current_class = old_current_class; - // Set the default include file name - if (!type->include().isValid()) { + // Set the default include file name. In case we saw an template instance earlier, + // overwrite the include file when we see the actual declaration. + if (!type->include().isValid() || class_item->hasActualDeclaration()) { QFileInfo info(class_item->fileName()); type->setInclude(Include(Include::IncludePath, info.fileName())); } @@ -1337,13 +1347,7 @@ bool AbstractMetaBuilder::setupInheritance(AbstractMetaClass *meta_class) TypeParser::Info info = TypeParser::parse(complete_name); QString base_name = info.qualified_name.join("::"); - AbstractMetaClass *templ = 0; - for (AbstractMetaClass *c : m_templates) { - if (c->typeEntry()->name() == base_name) { - templ = c; - break; - } - } + AbstractMetaClass *templ = m_templates.value(base_name); if (templ == 0) templ = m_meta_classes.findClass(base_name); diff --git a/generator/abstractmetabuilder.h b/generator/abstractmetabuilder.h index e93f45283..13ddf2857 100644 --- a/generator/abstractmetabuilder.h +++ b/generator/abstractmetabuilder.h @@ -150,7 +150,7 @@ class AbstractMetaBuilder QString m_file_name; AbstractMetaClassList m_meta_classes; - AbstractMetaClassList m_templates; + QHash m_templates; FileModelItem m_dom; QSet m_used_types; diff --git a/generator/abstractmetalang.h b/generator/abstractmetalang.h index 7ceda45d4..f161c5c78 100644 --- a/generator/abstractmetalang.h +++ b/generator/abstractmetalang.h @@ -675,6 +675,7 @@ class AbstractMetaClass : public AbstractMetaAttributes m_has_equals_operator(false), m_has_clone_operator(false), m_is_type_alias(false), + m_has_actual_declaration(false), m_qDebug_stream_function(0) { } @@ -796,6 +797,9 @@ class AbstractMetaClass : public AbstractMetaAttributes void setHasCloneOperator(bool on) { m_has_clone_operator = on; } bool hasCloneOperator() const { return m_has_clone_operator; } + void setHasActualDeclaration(bool on) { m_has_actual_declaration = on; } + bool hasActualDeclaration() const { return m_has_actual_declaration; } + QString getDefaultNonZeroFunction() const; void addPropertySpec(QPropertySpec *spec) { m_property_specs << spec; } @@ -856,7 +860,8 @@ class AbstractMetaClass : public AbstractMetaAttributes uint m_has_equals_operator : 1; uint m_has_clone_operator :1; uint m_is_type_alias : 1; - uint m_reserved : 18; + uint m_has_actual_declaration : 1; + uint m_reserved : 17; QString m_destructor_exception; const AbstractMetaClass *m_enclosing_class{}; diff --git a/generator/parser/binder.cpp b/generator/parser/binder.cpp index 0684eaac0..6f0e2adb6 100644 --- a/generator/parser/binder.cpp +++ b/generator/parser/binder.cpp @@ -729,6 +729,7 @@ void Binder::visitClassSpecifier(ClassSpecifierAST *node) name_cc.run(node->name->unqualified_name); _M_context.append(name_cc.name()); visitNodes(this, node->member_specs); + _M_current_class->setHasActualDeclaration(node->member_specs); _M_context.removeLast(); changeCurrentClass(old); diff --git a/generator/parser/codemodel.h b/generator/parser/codemodel.h index 3ee5d125d..71d8ed5bb 100644 --- a/generator/parser/codemodel.h +++ b/generator/parser/codemodel.h @@ -366,6 +366,10 @@ class _ClassModelItem: public _ScopeModelItem void addPropertyDeclaration(const QString &propertyDeclaration); QStringList propertyDeclarations() const { return _M_propertyDeclarations; } + void setHasActualDeclaration(bool flag) { _M_hasActualDeclaration = flag; } + bool hasActualDeclaration() const { return _M_hasActualDeclaration; } + + protected: _ClassModelItem(CodeModel *model, int kind = __node_kind) : _ScopeModelItem(model, kind), _M_classType(CodeModel::Class) {} @@ -377,6 +381,8 @@ class _ClassModelItem: public _ScopeModelItem QStringList _M_propertyDeclarations; + bool _M_hasActualDeclaration{}; + private: _ClassModelItem(const _ClassModelItem &other); void operator = (const _ClassModelItem &other); diff --git a/generator/typeparser.h b/generator/typeparser.h index b6ca9c573..20edb03f3 100644 --- a/generator/typeparser.h +++ b/generator/typeparser.h @@ -54,7 +54,7 @@ class TypeParser Info() : is_reference(false), is_constant(false), is_busted(false), indirections(0) { } QStringList qualified_name; QStringList arrays; - QList template_instantiations; + QList template_instantiations; // I believe template_argument_types would be a better name uint is_reference : 1; uint is_constant : 1; uint is_busted : 1; From 5d345bb705fd047239f0f5d7fb33b538d79fcf48 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Wed, 8 Nov 2023 13:37:38 +0100 Subject: [PATCH 13/21] Don't generate wrappers for templated methods in non-template classes --- generator/abstractmetabuilder.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index cfae4fa27..1c3d55a11 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -1491,6 +1491,12 @@ AbstractMetaFunction *AbstractMetaBuilder::traverseFunction(FunctionModelItem fu return 0; } + // Also filter out functions with template parameters in classes without template arguments + // (we don't support templated classes directly, but derived classes might derive from template instantiations) + if (function_item->templateParameters().size() && m_current_class->templateArguments().empty()) { + return 0; + } + if (function_item->isAuto()) { /*TODO: it might work just to output 'auto', but this would require * understanding what AbstractMetabuild::translateType() does and From dd8f906e2f4826e49cb9d1bc8f78aab89caa6200 Mon Sep 17 00:00:00 2001 From: mrbean-bremen Date: Wed, 1 Nov 2023 07:49:09 +0100 Subject: [PATCH 14/21] Fix handling of right-shift as template bracket - two consecutive right angle brackets had been handled as right-shift operator - replace right-shift with two brackets in parser where the context is clear --- generator/parser/lexer.cpp | 6 ++++-- generator/parser/parser.cpp | 33 +++++++++++++++++++++++++++++---- generator/parser/parser.h | 1 + generator/parser/tokens.cpp | 4 +++- generator/parser/tokens.h | 4 +++- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/generator/parser/lexer.cpp b/generator/parser/lexer.cpp index 2d40a706f..1c39aa0e0 100644 --- a/generator/parser/lexer.cpp +++ b/generator/parser/lexer.cpp @@ -624,7 +624,7 @@ void Lexer::scan_less() } else { - token_stream[(int) index++].kind = Token_shift; + token_stream[(int) index++].kind = Token_shift_left; } } else @@ -677,7 +677,9 @@ void Lexer::scan_greater() } else { - token_stream[(int) index++].kind = Token_shift; + // may be replaced by two ">" during parsing + token_stream[(int) index++].kind = Token_shift_right; + token_stream[(int) index++].kind = Token_placeholder; } } else diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index f2b6f5f35..4cfb0edd8 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -896,6 +896,7 @@ bool Parser::parseTemplateDeclaration(DeclarationAST *&node) nextToken(); parseTemplateParameterList(params); + resolveRightShift(); ADVANCE('>', ">"); } @@ -957,7 +958,7 @@ bool Parser::parseOperator(OperatorAST *&node) case '>': case ',': case Token_assign: - case Token_shift: + case Token_shift_left: case Token_eq: case Token_not_eq: case Token_leq: @@ -972,6 +973,13 @@ bool Parser::parseOperator(OperatorAST *&node) nextToken(); break; + case Token_shift_right: + ast->op = token_stream.cursor(); + nextToken(); + // skip placeholder + nextToken(); + break; + default: if (token_stream.lookAhead() == '(' && token_stream.lookAhead(1) == ')') @@ -1157,7 +1165,8 @@ bool Parser::parseTemplateArgument(TemplateArgumentAST *&node) ExpressionAST *expr = 0; if (!parseTypeId(typeId) || (token_stream.lookAhead() != ',' - && token_stream.lookAhead() != '>')) + && token_stream.lookAhead() != '>' + && token_stream.lookAhead() != Token_shift_right)) { token_stream.rewind((int) start); @@ -2553,6 +2562,16 @@ bool Parser::parsePtrToMember(PtrToMemberAST *&node) return false; } +void Parser::resolveRightShift() +{ + if (token_stream.lookAhead() == Token_shift_right) + { + int index = token_stream.cursor(); + token_stream[index].kind = '>'; + token_stream[index + 1].kind = '>'; + } +} + bool Parser::parseUnqualifiedName(UnqualifiedNameAST *&node, bool parseTemplateId) { @@ -2602,6 +2621,7 @@ bool Parser::parseUnqualifiedName(UnqualifiedNameAST *&node, // optional template arguments parseTemplateArgumentList(ast->template_arguments); + resolveRightShift(); if (token_stream.lookAhead() == '>') { nextToken(); @@ -2609,7 +2629,7 @@ bool Parser::parseUnqualifiedName(UnqualifiedNameAST *&node, else { ast->template_arguments = 0; - token_stream.rewind((int) index); + token_stream.rewind((int)index); } } } @@ -4207,9 +4227,14 @@ bool Parser::parseShiftExpression(ExpressionAST *&node) if (!parseAdditiveExpression(node)) return false; - while (token_stream.lookAhead() == Token_shift) + while (token_stream.lookAhead() == Token_shift_left || token_stream.lookAhead() == Token_shift_right) { std::size_t op = token_stream.cursor(); + if (token_stream.lookAhead() == Token_shift_right) + { + // skip placeholder + nextToken(); + } nextToken(); ExpressionAST *rightExpr = 0; diff --git a/generator/parser/parser.h b/generator/parser/parser.h index 36d77c57c..461323585 100644 --- a/generator/parser/parser.h +++ b/generator/parser/parser.h @@ -147,6 +147,7 @@ class Parser bool parsePrimaryExpression(ExpressionAST *&node); bool parsePtrOperator(PtrOperatorAST *&node); bool parsePtrToMember(PtrToMemberAST *&node); + void resolveRightShift(); bool parseRelationalExpression(ExpressionAST *&node, bool templArgs = false); bool parseShiftExpression(ExpressionAST *&node); diff --git a/generator/parser/tokens.cpp b/generator/parser/tokens.cpp index a1a39c5c1..a7b8d0c63 100644 --- a/generator/parser/tokens.cpp +++ b/generator/parser/tokens.cpp @@ -113,6 +113,7 @@ static char const * const _S_token_names[] = { "operator", "or", "or_eq", + "placeholder", "preproc", "private", "protected", @@ -122,7 +123,8 @@ static char const * const _S_token_names[] = { "reinterpret_cast", "return", "scope", - "shift", + "shift_left", + "shift_right", "short", "signals", "signed", diff --git a/generator/parser/tokens.h b/generator/parser/tokens.h index 034d915be..416e0a9a6 100644 --- a/generator/parser/tokens.h +++ b/generator/parser/tokens.h @@ -114,6 +114,7 @@ enum TOKEN_KIND Token_operator, Token_or, Token_or_eq, + Token_placeholder, Token_preproc, Token_private, Token_protected, @@ -123,7 +124,8 @@ enum TOKEN_KIND Token_reinterpret_cast, Token_return, Token_scope, - Token_shift, + Token_shift_left, + Token_shift_right, Token_short, Token_signals, Token_signed, From d129faf860e97c0b048cdb8690168d840f432670 Mon Sep 17 00:00:00 2001 From: mrbean-bremen Date: Wed, 1 Nov 2023 07:52:07 +0100 Subject: [PATCH 15/21] Ignore variadic template parameters - ignore default template parameters --- generator/parser/parser.cpp | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index 4cfb0edd8..d1153b58f 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -466,6 +466,9 @@ bool Parser::parseDeclaration(DeclarationAST *&node) switch(token_stream.lookAhead()) { case ';': + // ignore ellipse in constructs like + // using T1 = T2...>; + case Token_ellipsis: nextToken(); return true; @@ -1543,6 +1546,17 @@ bool Parser::parseTemplateParameterList(const ListNode *& } } + if (token_stream.lookAhead() == Token_ellipsis) + { + // ignore variadic template parameters as in + // template + token_stream.nextToken(); + if (token_stream.lookAhead() == Token_identifier) + { + token_stream.nextToken(); + } + } + return true; } @@ -1556,7 +1570,13 @@ bool Parser::parseTemplateParameter(TemplateParameterAST *&node) if ((tk == Token_class || tk == Token_typename || tk == Token_template) && parseTypeParameter(ast->type_parameter)) { - // nothing to do + if (token_stream.lookAhead() == '=' || token_stream.lookAhead() == Token_assign) + { + // default template parameter + token_stream.nextToken(); + TypeSpecifierAST* typeNode = 0; + parseTypeSpecifier(typeNode); + } } else if (!parseParameterDeclaration(ast->parameter_declaration)) return false; @@ -1581,6 +1601,12 @@ bool Parser::parseTypeParameter(TypeParameterAST *&node) { nextToken(); // skip class + if (token_stream.lookAhead() == Token_ellipsis) + { + // skip variadic parameters in template parameter list + nextToken(); + } + // parse optional name if(parseName(ast->name, true)) { @@ -2252,6 +2278,13 @@ bool Parser::parseInitDeclarator(InitDeclaratorAST *&node) { std::size_t start = token_stream.cursor(); + if (token_stream.lookAhead(0) == Token_ellipsis) + { + // for now just ignore variadic parameters + token_stream.nextToken(); + return true; + } + DeclaratorAST *decl = 0; if (!parseDeclarator(decl)) { From 164969deb4b174c90076213e90d889bc3a50678d Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Thu, 9 Nov 2023 18:17:30 +0100 Subject: [PATCH 16/21] Handle ellipsis in type parser --- generator/typeparser.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/generator/typeparser.cpp b/generator/typeparser.cpp index 87730fa5c..90a68e568 100644 --- a/generator/typeparser.cpp +++ b/generator/typeparser.cpp @@ -53,6 +53,7 @@ class Scanner LessThanToken, ColonToken, CommaToken, + EllipsisToken, OpenParenToken, CloseParenToken, SquareBegin, @@ -115,6 +116,12 @@ Scanner::Token Scanner::nextToken() Q_ASSERT(m_pos + 1 < m_length); ++m_pos; break; + case '.': + if (m_pos + 2 < m_length && c == m_chars[m_pos + 1] && c == m_chars[m_pos + 2]) { + tok = EllipsisToken; + m_pos += 2; + break; + } default: if (c.isLetterOrNumber() || c == '_') tok = Identifier; @@ -168,6 +175,7 @@ TypeParser::Info TypeParser::parse(const QString &str) // switch (tok) { // case Scanner::StarToken: printf(" - *\n"); break; // case Scanner::AmpersandToken: printf(" - &\n"); break; +// case Scanner::EllipsisToken: printf(" - ...\n"); break; // case Scanner::LessThanToken: printf(" - <\n"); break; // case Scanner::GreaterThanToken: printf(" - >\n"); break; // case Scanner::ColonToken: printf(" - ::\n"); break; @@ -215,7 +223,8 @@ TypeParser::Info TypeParser::parse(const QString &str) case Scanner::OpenParenToken: // function pointers not supported case Scanner::CloseParenToken: - { + case Scanner::EllipsisToken: // variadic templates not supported + { Info i; i.is_busted = true; return i; From 393af7368f2b322f3b50d731d4d4f72987480b20 Mon Sep 17 00:00:00 2001 From: John Bowler Date: Fri, 10 Nov 2023 05:25:02 -0800 Subject: [PATCH 17/21] Fix: AbstractMetaBuilder::classesTopologicalSorted (#148) This explicit new/delete, storage leaks (missing delete) and iteration over a QHash while modifying it (deleting specific keys in an iterator). --- generator/abstractmetabuilder.cpp | 153 ++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 48 deletions(-) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index 1c3d55a11..37d800c8b 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -2501,55 +2501,112 @@ void AbstractMetaBuilder::dumpLog() AbstractMetaClassList AbstractMetaBuilder::classesTopologicalSorted() const { - AbstractMetaClassList res; - - AbstractMetaClassList classes = m_meta_classes; - classes.sort(); - - QSet noDependency; - QHash* > hash; - for (AbstractMetaClass *cls : classes) { - QSet *depends = new QSet(); + /* This function is the standard topological sort of a Directed Acyclic + * Graph (a DAG). It outputs a partially ordered list of the nodes in the + * graph such that a node is output after all of its children. + * + * In the previous implementation it seemed to account for around 68-69% of + * the entire run time of pythonqt_generator however this might be an + * artefact of the profiling technique used as these changes make not + * significant difference to the run time of the generator. + * + * However the previous implementation also leaked memory and modified a + * QHash while iterating over it with a QHashIterator; a potentially fatal + * "use after free". + * + * This version is somewhat like the python equivalent using list + * comprehensions. + */ + + /* Build a hash list of QSet for each class. 'class' is represented + * by a pointer to the AbstractMetaClass from m_meta_classes. + */ + ReportHandler::debugSparse(QString("TSORT: %1 meta classes") + .arg(m_meta_classes.count())); + QHash> classes; + classes.reserve(m_meta_classes.count()); + + for (auto cls : m_meta_classes) { + /* Add the baseClass and the interfaces the class uses. The latter + * are stored in an AbstractMetaClassList which is uses a QList, so: + */ + auto entry(classes.insert(cls, +# if QT_VERSION < QT_VERSION_CHECK(5,14,0) + QSet::fromList(cls->interfaces()) +# else + QSet(cls->interfaces().cbegin(), + cls->interfaces().cend()) +# endif + )); if (cls->baseClass()) - depends->insert(cls->baseClass()); - - for (AbstractMetaClass *interface : cls->interfaces()) { - AbstractMetaClass *impl = interface->primaryInterfaceImplementor(); - if (impl == cls) - continue; - depends->insert(impl); - } - - if (depends->empty()) { - noDependency.insert(cls); - } else { - hash.insert(cls, depends); - } - } - - while (!noDependency.empty()) { - for (AbstractMetaClass *cls : noDependency.values()) { - if(!cls->isInterface()) - res.append(cls); - noDependency.remove(cls); - QHashIterator* > i(hash); - while (i.hasNext()) { - i.next(); - i.value()->remove(cls); - if (i.value()->empty()) { - AbstractMetaClass *key = i.key(); - noDependency.insert(key); - hash.remove(key); - delete(i.value()); - } - } - } - } - - if (!noDependency.empty() || !hash.empty()) { - qWarning("dependency graph was cyclic."); - } + entry.value().insert(cls->baseClass()); + entry.value().remove(cls); // may come from interfaces + } + + /* This and the qFatals below are fatal internal errors in the code of + * pythonqt_generator. + */ + if (m_meta_classes.count() != classes.count()) + qFatal("TOPO SORT: duplicate meta classes (%lld != %lld)", + static_cast(m_meta_classes.count()), + static_cast(classes.count())); + + /* Loop: output all the classes with no remaining dependencies to the + * result and then also remove those now output classes from the remaining + * classes in the hash table. + */ + AbstractMetaClassList result; + result.reserve(classes.count()); + QSet handled; + handled.reserve(classes.count()); + int interfaceClasses(0), depthFromLeaf(0); + + while (!classes.empty()) { + handled.clear(); + int iFCount(0); + + /* Output classes where all children have already been output (initially + * the leaf nodes): + */ + for (auto i(classes.cbegin()); i != classes.cend(); ++i) + if (i.value().empty()) + handled.insert(i.key()); - return res; + /* Something must have been done; if not this is not a DAG because + * there is a cycle. + */ + if (handled.empty()) + qFatal("TOPOSORT: %lld cyclic meta classes @depth %d.", + static_cast(classes.count()), depthFromLeaf); + + /* Remove all 'handled' from the hash table. */ + for (auto cls : handled) + if (!classes.remove(cls)) + qFatal("TOPO SORT: class remove failed @depth %d.", + depthFromLeaf); + + /* Then remove the 'handled' set from the classes values: */ + for (QSet &set : classes) + set -= handled; + + /* Output only those handled classes there are not interfaces: */ + for (auto cls : handled) { + if (!cls->isInterface()) + result.append(cls); + else + ++iFCount; + } + + ReportHandler::debugSparse( + QString("TSORT: depth %1: %2 classes (%3 interface)") + .arg(depthFromLeaf).arg(handled.count()).arg(iFCount)); + interfaceClasses += iFCount; + ++depthFromLeaf; + } + + ReportHandler::debugSparse(QString( + "TSORT: %1 result classes, %2 interface classes)") + .arg(result.count()).arg(interfaceClasses)); + return result; } From 73993d4a0b62173ea44509614fee420f7028bb1c Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Fri, 10 Nov 2023 12:25:06 +0100 Subject: [PATCH 18/21] Improve debugging code for rewind, also show identifiers --- generator/parser/lexer.h | 2 +- generator/parser/parser.cpp | 118 ++++++++++++++++++++---------------- generator/parser/parser.h | 2 + 3 files changed, 69 insertions(+), 53 deletions(-) diff --git a/generator/parser/lexer.h b/generator/parser/lexer.h index 1a1097ec4..4d4405365 100644 --- a/generator/parser/lexer.h +++ b/generator/parser/lexer.h @@ -140,7 +140,7 @@ class TokenStream inline std::size_t cursor() const { return index; } - inline void rewind(int i) + inline void rewind(std::size_t i) { index = i; } void resize(std::size_t size) diff --git a/generator/parser/parser.cpp b/generator/parser/parser.cpp index d1153b58f..bdf39f76f 100644 --- a/generator/parser/parser.cpp +++ b/generator/parser/parser.cpp @@ -102,6 +102,12 @@ void Parser::nextToken() keepTrackDebug(); } +void Parser::rewind(std::size_t pos) +{ + token_stream.rewind(pos); + keepTrackDebug(); +} + TranslationUnitAST *Parser::parse(const char *contents, std::size_t size, pool *p) { @@ -213,6 +219,14 @@ void Parser::keepTrackDebug() location().positionAt(token_stream.position(tok), &_currentLine, &_currentColumn, &_currentFile); _currentToken = token_name(token_stream.lookAhead()); + if (token_stream.lookAhead() == Token_identifier) + { + _currentSymbol = token_stream.symbol(tok)->as_string(); + } + else + { + _currentSymbol = ""; + } #endif } @@ -407,7 +421,7 @@ bool Parser::parseName(NameAST *&node, bool acceptTemplateId) Q_ASSERT(n != 0); if (!acceptTemplateId) { - token_stream.rewind((int) n->start_token); + rewind(n->start_token); parseUnqualifiedName(n, false); } @@ -534,7 +548,7 @@ bool Parser::parseDeclaration(DeclarationAST *&node) } } // end switch - token_stream.rewind((int) start); + rewind(start); return parseDeclarationInternal(node); } @@ -1082,7 +1096,7 @@ bool Parser::parseSimpleTypeSpecifier(TypeSpecifierAST *&node, if (token_stream.lookAhead() != ')') { ast->type_id = 0; - token_stream.rewind((int) saved); + rewind(saved); parseUnaryExpression(ast->expression); } ADVANCE(')', ")"); @@ -1094,7 +1108,7 @@ bool Parser::parseSimpleTypeSpecifier(TypeSpecifierAST *&node, } else if (onlyIntegral) { - token_stream.rewind((int) start); + rewind(start); return false; } else @@ -1102,7 +1116,7 @@ bool Parser::parseSimpleTypeSpecifier(TypeSpecifierAST *&node, if (!parseName(ast->name, true)) { ast->name = 0; - token_stream.rewind((int) start); + rewind(start); return false; } } @@ -1141,7 +1155,7 @@ bool Parser::parsePtrOperator(PtrOperatorAST *&node) { if (!parsePtrToMember(ast->mem_ptr)) { - token_stream.rewind((int) start); + rewind(start); return false; } } @@ -1171,7 +1185,7 @@ bool Parser::parseTemplateArgument(TemplateArgumentAST *&node) && token_stream.lookAhead() != '>' && token_stream.lookAhead() != Token_shift_right)) { - token_stream.rewind((int) start); + rewind(start); if (!parseLogicalOrExpression(expr, true)) return false; @@ -1197,7 +1211,7 @@ bool Parser::parseTypeSpecifier(TypeSpecifierAST *&node) TypeSpecifierAST *ast = 0; if (!parseElaboratedTypeSpecifier(ast) && !parseSimpleTypeSpecifier(ast)) { - token_stream.rewind((int) start); + rewind(start); return false; } @@ -1247,7 +1261,7 @@ bool Parser::parseDeclarator(DeclaratorAST *&node) } else { - token_stream.rewind((int) start); + rewind(start); return false; } @@ -1294,7 +1308,7 @@ bool Parser::parseDeclarator(DeclaratorAST *&node) && !(isVector || tok == '(' || tok == ',' || tok == ';' || tok == '=')) { - token_stream.rewind((int) start); + rewind(start); return false; } @@ -1306,7 +1320,7 @@ bool Parser::parseDeclarator(DeclaratorAST *&node) ParameterDeclarationClauseAST *params = 0; if (!parseParameterDeclarationClause(params)) { - token_stream.rewind((int) index); + rewind(index); goto update_pos; } @@ -1314,7 +1328,7 @@ bool Parser::parseDeclarator(DeclaratorAST *&node) if (token_stream.lookAhead() != ')') { - token_stream.rewind((int) index); + rewind(index); goto update_pos; } @@ -1374,7 +1388,7 @@ bool Parser::parseAbstractDeclarator(DeclaratorAST *&node) if (!parseAbstractDeclarator(decl)) { - token_stream.rewind((int) index); + rewind(index); goto label1; } @@ -1382,7 +1396,7 @@ bool Parser::parseAbstractDeclarator(DeclaratorAST *&node) if (token_stream.lookAhead() != ')') { - token_stream.rewind((int) start); + rewind(start); return false; } nextToken(); @@ -1420,7 +1434,7 @@ bool Parser::parseAbstractDeclarator(DeclaratorAST *&node) && !(isVector || tok == '(' || tok == ',' || tok == ';' || tok == '=')) { - token_stream.rewind((int) start); + rewind(start); return false; } @@ -1432,7 +1446,7 @@ bool Parser::parseAbstractDeclarator(DeclaratorAST *&node) ParameterDeclarationClauseAST *params = 0; if (!parseParameterDeclarationClause(params)) { - token_stream.rewind((int) index); + rewind(index); goto update_pos; } @@ -1440,7 +1454,7 @@ bool Parser::parseAbstractDeclarator(DeclaratorAST *&node) if (token_stream.lookAhead() != ')') { - token_stream.rewind((int) index); + rewind(index); goto update_pos; } @@ -1481,14 +1495,14 @@ bool Parser::parseEnumSpecifier(TypeSpecifierAST *&node) TypeSpecifierAST *ast = 0; if (!parseSimpleTypeSpecifier(ast)) { - token_stream.rewind((int) start); + rewind(start); return false; } } if (token_stream.lookAhead() != '{') { - token_stream.rewind((int) start); + rewind(start); return false; } nextToken(); @@ -1617,14 +1631,14 @@ bool Parser::parseTypeParameter(TypeParameterAST *&node) if(!parseTypeId(ast->type_id)) { //syntaxError(); - token_stream.rewind((int) start); + rewind(start); return false; } } else if (token_stream.lookAhead() != ',' && token_stream.lookAhead() != '>') { - token_stream.rewind((int) start); + rewind(start); return false; } } @@ -1720,7 +1734,7 @@ bool Parser::parseTypeId(TypeIdAST *&node) TypeSpecifierAST *spec = 0; if (!parseTypeSpecifier(spec)) { - token_stream.rewind((int) start); + rewind(start); return false; } @@ -1804,7 +1818,7 @@ bool Parser::parseParameterDeclarationList(const ListNode *cv = 0; parseCvQualify(cv); @@ -2148,7 +2162,7 @@ bool Parser::parseMemberSpecification(DeclarationAST *&node) return true; } - token_stream.rewind((int) start); + rewind(start); return parseDeclarationInternal(node); } @@ -2202,7 +2216,7 @@ bool Parser::parseElaboratedTypeSpecifier(TypeSpecifierAST *&node) } } - token_stream.rewind((int) start); + rewind(start); return false; } @@ -2591,7 +2605,7 @@ bool Parser::parsePtrToMember(PtrToMemberAST *&node) nextToken(); } - token_stream.rewind((int) start); + rewind(start); return false; } @@ -2599,7 +2613,7 @@ void Parser::resolveRightShift() { if (token_stream.lookAhead() == Token_shift_right) { - int index = token_stream.cursor(); + std::size_t index = token_stream.cursor(); token_stream[index].kind = '>'; token_stream[index + 1].kind = '>'; } @@ -2662,7 +2676,7 @@ bool Parser::parseUnqualifiedName(UnqualifiedNameAST *&node, else { ast->template_arguments = 0; - token_stream.rewind((int)index); + rewind(index); } } } @@ -2798,7 +2812,7 @@ bool Parser::parseExpressionOrDeclarationStatement(StatementAST *&node) std::size_t end = token_stream.cursor(); - token_stream.rewind((int) start); + rewind(start); StatementAST *expr_ast = 0; maybe_amb &= parseExpressionStatement(expr_ast); maybe_amb &= token_stream.kind(token_stream.cursor() - 1) == ';'; @@ -2816,7 +2830,7 @@ bool Parser::parseExpressionOrDeclarationStatement(StatementAST *&node) } else { - token_stream.rewind((int) std::max(end, token_stream.cursor())); + rewind(std::max(end, token_stream.cursor())); node = decl_ast; if (!node) @@ -2847,7 +2861,7 @@ bool Parser::parseCondition(ConditionAST *&node, bool initRequired) DeclaratorAST *decl = 0; if (!parseDeclarator(decl)) { - token_stream.rewind((int) declarator_start); + rewind(declarator_start); if (!initRequired && !parseAbstractDeclarator(decl)) decl = 0; } @@ -2870,7 +2884,7 @@ bool Parser::parseCondition(ConditionAST *&node, bool initRequired) } } - token_stream.rewind((int) start); + rewind(start); if (!parseCommaExpression(ast->expression)) return false; @@ -3191,7 +3205,7 @@ bool Parser::parseBlockDeclaration(DeclarationAST *&node) TypeSpecifierAST *spec = 0; if (!parseTypeSpecifierOrClassSpec(spec)) { // replace with simpleTypeSpecifier?!?! - token_stream.rewind((int) start); + rewind(start); return false; } @@ -3203,7 +3217,7 @@ bool Parser::parseBlockDeclaration(DeclarationAST *&node) if (token_stream.lookAhead() != ';') { - token_stream.rewind((int) start); + rewind(start); return false; } nextToken(); @@ -3298,7 +3312,7 @@ bool Parser::parseDeclarationInternal(DeclarationAST *&node) { // no type specifier, maybe a constructor or a cast operator?? - token_stream.rewind((int) index); + rewind(index); InitDeclaratorAST *declarator = 0; if (parseInitDeclarator(declarator)) @@ -3378,7 +3392,7 @@ bool Parser::parseDeclarationInternal(DeclarationAST *&node) } start_decl: - token_stream.rewind((int) index); + rewind(index); if ((token_stream.lookAhead() == Token_const || token_stream.lookAhead() == Token_constexpr) @@ -3433,7 +3447,7 @@ bool Parser::parseDeclarationInternal(DeclarationAST *&node) } else { - token_stream.rewind((int) startDeclarator); + rewind(startDeclarator); if (!parseInitDeclaratorList(declarators)) { syntaxError(); @@ -3710,7 +3724,7 @@ bool Parser::parsePostfixExpressionInternal(ExpressionAST *&node) nextToken(); } - int saved = int(token_stream.cursor()); + std::size_t saved = token_stream.cursor(); NameAST *name = 0; if (parseName(name, true) && name->unqualified_name @@ -3719,7 +3733,7 @@ bool Parser::parsePostfixExpressionInternal(ExpressionAST *&node) // a template method call // ### reverse the logic } else { - token_stream.rewind(saved); + rewind(saved); name = 0; if (! parseName (name, templ != 0)) @@ -3862,14 +3876,14 @@ bool Parser::parsePostfixExpression(ExpressionAST *&node) if (parseCastExpression(cast_expr) && cast_expr->kind == AST::Kind_CastExpression) { - token_stream.rewind((int) saved_pos); + rewind(saved_pos); parsePrimaryExpression(expr); goto L_no_rewind; } } } - token_stream.rewind((int) saved_pos); + rewind(saved_pos); L_no_rewind: if (!expr && parseSimpleTypeSpecifier(typeSpec) @@ -3886,7 +3900,7 @@ bool Parser::parsePostfixExpression(ExpressionAST *&node) else { typeSpec = 0; - token_stream.rewind((int) start); + rewind(start); if (!parsePrimaryExpression(expr)) return false; @@ -3967,7 +3981,7 @@ bool Parser::parseUnaryExpression(ExpressionAST *&node) } ast->type_id = 0; - token_stream.rewind((int) index); + rewind(index); } if (!parseUnaryExpression(ast->expression)) @@ -4162,7 +4176,7 @@ bool Parser::parseCastExpression(ExpressionAST *&node) } } - token_stream.rewind((int) start); + rewind(start); return parseUnaryExpression(node); } diff --git a/generator/parser/parser.h b/generator/parser/parser.h index 461323585..92d87ed3a 100644 --- a/generator/parser/parser.h +++ b/generator/parser/parser.h @@ -191,6 +191,7 @@ class Parser bool skip(int l, int r); void nextToken(); + void rewind(std::size_t pos); // private: TokenStream token_stream; @@ -213,6 +214,7 @@ class Parser int _currentLine{}; int _currentColumn{}; const char* _currentToken{}; + QString _currentSymbol; private: Parser(const Parser& source); From 6fdafa17397a8f79f8d10e76a717c791d3b630d3 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Wed, 15 Nov 2023 19:08:44 +0100 Subject: [PATCH 19/21] Add qsizetype and qptrdiff as primitive types --- generator/typesystem_core.xml | 2 ++ src/PythonQtMethodInfo.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/generator/typesystem_core.xml b/generator/typesystem_core.xml index 26b36663b..d21f70808 100644 --- a/generator/typesystem_core.xml +++ b/generator/typesystem_core.xml @@ -201,6 +201,8 @@ + + diff --git a/src/PythonQtMethodInfo.cpp b/src/PythonQtMethodInfo.cpp index 4344be909..60c996dc1 100644 --- a/src/PythonQtMethodInfo.cpp +++ b/src/PythonQtMethodInfo.cpp @@ -318,15 +318,19 @@ int PythonQtMethodInfo::nameToType(const char* name) _parameterTypeDict.insert("qgl_GLsizeiptr", QMetaType::LongLong); _parameterTypeDict.insert("size_t", QMetaType::ULongLong); _parameterTypeDict.insert("qintptr", QMetaType::LongLong); + _parameterTypeDict.insert("qptrdiff", QMetaType::LongLong); _parameterTypeDict.insert("quintptr", QMetaType::ULongLong); _parameterTypeDict.insert("WId", QMetaType::ULongLong); + _parameterTypeDict.insert("qsizetype", QMetaType::LongLong); } else { _parameterTypeDict.insert("qgl_GLintptr", QMetaType::Int); _parameterTypeDict.insert("qgl_GLsizeiptr", QMetaType::Int); _parameterTypeDict.insert("size_t", QMetaType::UInt); _parameterTypeDict.insert("qintptr", QMetaType::Int); + _parameterTypeDict.insert("qptrdiff", QMetaType::Int); _parameterTypeDict.insert("quintptr", QMetaType::UInt); _parameterTypeDict.insert("WId", QMetaType::UInt); + _parameterTypeDict.insert("qsizetype", QMetaType::Int); } #ifdef PYTHONQT_SUPPORT_ML_TYPES From e80459435e91caa8eeb3e94901e303d90965d1c0 Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Tue, 14 Nov 2023 15:53:10 +0100 Subject: [PATCH 20/21] Avoid crash if supposed property setter has no arguments --- generator/abstractmetabuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/abstractmetabuilder.cpp b/generator/abstractmetabuilder.cpp index 37d800c8b..e7cb889b9 100644 --- a/generator/abstractmetabuilder.cpp +++ b/generator/abstractmetabuilder.cpp @@ -1253,7 +1253,7 @@ void AbstractMetaBuilder::traverseFunctions(ScopeModelItem scope_item, AbstractM } } else if (QPropertySpec *write = meta_class->propertySpecForWrite(meta_function->name())) { - if (write->type() == meta_function->arguments().at(0)->type()->typeEntry()) { + if (meta_function->arguments().size() == 1 && write->type() == meta_function->arguments().at(0)->type()->typeEntry()) { *meta_function += AbstractMetaAttributes::PropertyWriter; meta_function->setPropertySpec(write); // printf("%s is writer for %s\n", From 2aecabb6c961a6c4a51f0ed9817945caec3b756a Mon Sep 17 00:00:00 2001 From: Uwe Siems Date: Tue, 14 Nov 2023 17:41:53 +0100 Subject: [PATCH 21/21] Avoid crash in lexer by increasing size of token_stream early The previous code assumed that any parsing step would only add one token at a time, which isn't true. This would lead to crashes if the size doubling threshold was skipped at one point. --- generator/parser/lexer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/parser/lexer.cpp b/generator/parser/lexer.cpp index 1c39aa0e0..db614ad17 100644 --- a/generator/parser/lexer.cpp +++ b/generator/parser/lexer.cpp @@ -142,7 +142,7 @@ void Lexer::tokenize(const char *contents, std::size_t size) line_table.current_line = 1; do { - if (index == token_stream.size()) + if (index >= token_stream.size()-4) // increase size of token_stream early, in case more than one token is written in one go token_stream.resize(token_stream.size() * 2); Token *current_token = &token_stream[(int) index];