diff --git a/docs/_docs/code-snippets/dotnet/JobScheduling.cs b/docs/_docs/code-snippets/dotnet/JobScheduling.cs new file mode 100644 index 00000000000000..cd11e61df8ab73 --- /dev/null +++ b/docs/_docs/code-snippets/dotnet/JobScheduling.cs @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Apache.Ignite.Core; +using Apache.Ignite.Core.Compute; +using Apache.Ignite.Core.Resource; + +namespace dotnet_helloworld +{ + public class JobScheduling + { + public void Priority() + { + // tag::priority[] + // PriorityQueueCollisionSpi must be configured in the Spring XML configuration file ignite-helloworld.xml + var cfg = new IgniteConfiguration + { + SpringConfigUrl = "ignite-helloworld.xml" + }; + + // Start a node. + using var ignite = Ignition.Start(cfg); + // end::priority[] + } + + // tag::task-priority[] + // Compute tasks must be annotated with the ComputeTaskSessionFullSupport attribute to support distributing + // the task's session attributes to compute jobs that the task creates. + [ComputeTaskSessionFullSupport] + public class MyUrgentTask : ComputeTaskSplitAdapter + { + // Auto-injected task session. + [TaskSessionResource] private IComputeTaskSession _taskSes; + + /// + protected override ICollection> Split(int gridSize, int arg) + { + // Set high task priority. + _taskSes.SetAttribute("grid.task.priority", 10); + + var jobs = new List>(gridSize); + + for (var i = 1; i <= gridSize; i++) + { + jobs.Add(new MyUrgentJob()); + } + + // These jobs will be executed with higher priority. + return jobs; + } + + /// + public override bool Reduce(IList> results) => results.All(r => r.Data); + } + // end::task-priority[] + + private class MyUrgentJob : ComputeJobAdapter + { + public override bool Execute() => true; + } + } +} \ No newline at end of file diff --git a/docs/_docs/distributed-computing/job-scheduling.adoc b/docs/_docs/distributed-computing/job-scheduling.adoc index c242a69c44ce05..fffde3274bd9da 100644 --- a/docs/_docs/distributed-computing/job-scheduling.adoc +++ b/docs/_docs/distributed-computing/job-scheduling.adoc @@ -15,6 +15,7 @@ = Job Scheduling :javaFile: {javaCodeDir}/JobScheduling.java +:csharpFile: {csharpCodeDir}/JobScheduling.cs When jobs arrive at the destination node, they are submitted to a thread pool and scheduled for execution in random order. However, you can change job ordering by configuring `CollisionSpi`. @@ -70,9 +71,16 @@ tab:C++[unsupported] Task priorities are set in the link:distributed-computing/map-reduce#distributed-task-session[task session] via the `grid.task.priority` attribute. If no priority is assigned to a task, then the default priority of 0 is used. - +[tabs] +-- +tab:Java[] [source, java] ---- include::{javaFile}[tag=task-priority,indent=0] ---- - +tab:C#/.NET[] +[source,csharp] +---- +include::{csharpFile}[tag=task-priority,indent=0] +---- +-- diff --git a/docs/_docs/extensions-and-integrations/change-data-capture-extensions.adoc b/docs/_docs/extensions-and-integrations/change-data-capture-extensions.adoc index 399643e96adb2f..5973b34607eede 100644 --- a/docs/_docs/extensions-and-integrations/change-data-capture-extensions.adoc +++ b/docs/_docs/extensions-and-integrations/change-data-capture-extensions.adoc @@ -241,6 +241,19 @@ You should consider the nature of your transactions, the rate of change of your Custom conflict resolver can be set via `conflictResolver` and allows to compare or merge the conflict data in any required way. +=== Conflict Resolver Metrics + +The Ignite's built-in `CacheVersionConflictResolverPluginProvider` provides the following metrics: + +[cols="35%,65%",opts="header"] +|=== +|Name |Description +| `AcceptedCount` | Count of accepted entries. +| `RejectedCount` | Count of rejected entries. +|=== + +These metrics are registered under `conflict-resolver` registry for each node configured with this plugin. + === Configuration example Configuration is done via Ignite node plugin: diff --git a/modules/bom/pom.xml b/modules/bom/pom.xml index 40ea220bdd2a1e..32d6659279a4b8 100644 --- a/modules/bom/pom.xml +++ b/modules/bom/pom.xml @@ -116,6 +116,11 @@ ignite-opencensus ${revision} + + ${project.groupId} + ignite-json + ${revision} + ${project.groupId} ignite-rest-http diff --git a/modules/calcite/pom.xml b/modules/calcite/pom.xml index 9aef74c606a3db..57ee3a8013e1b3 100644 --- a/modules/calcite/pom.xml +++ b/modules/calcite/pom.xml @@ -42,7 +42,7 @@ 2.8.2 3.1.8 2.4 - 2.7.0 + 2.9.0 0.10.2 3.6.1 diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ConverterUtils.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ConverterUtils.java index 4f18bd6f5e2e15..0beb119701174b 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ConverterUtils.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ConverterUtils.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.linq4j.tree.ConstantExpression; import org.apache.calcite.linq4j.tree.ConstantUntypedNull; @@ -221,14 +220,14 @@ public static Expression convert(Expression operand, Type fromType, Type toType) final Primitive fromBox = Primitive.ofBox(fromType); final Primitive fromPrimitive = Primitive.of(fromType); final boolean fromNumber = fromType instanceof Class - && Number.class.isAssignableFrom((Class)fromType); + && Number.class.isAssignableFrom((Class)fromType); if (fromType == String.class) { if (toPrimitive != null) { + if (toPrimitive.isFixedNumeric()) + return IgniteExpressions.parseStringChecked(operand, toPrimitive); + switch (toPrimitive) { case CHAR: - case SHORT: - case INT: - case LONG: case FLOAT: case DOUBLE: // Generate "SqlFunctions.toShort(x)". @@ -245,6 +244,9 @@ public static Expression convert(Expression operand, Type fromType, Type toType) } } if (toBox != null) { + if (toBox.isFixedNumeric()) + operand = IgniteExpressions.parseStringChecked(operand, toBox); + switch (toBox) { case CHAR: // Generate "SqlFunctions.toCharBoxed(x)". @@ -277,12 +279,11 @@ public static Expression convert(Expression operand, Type fromType, Type toType) if (fromPrimitive != null) { // E.g. from "float" to "double" - return Expressions.convert_( - operand, toPrimitive.primitiveClass); + return IgniteExpressions.convertChecked(operand, fromPrimitive, toPrimitive); } - if (fromNumber || fromBox == Primitive.CHAR) { + if (fromNumber) { // Generate "x.shortValue()". - return Expressions.unbox(operand, toPrimitive); + return IgniteExpressions.unboxChecked(operand, fromBox, toPrimitive); } else { // E.g. from "Object" to "short". @@ -300,7 +301,7 @@ else if (fromNumber && toBox != null) { Expressions.equal(operand, RexImpTable.NULL_EXPR), RexImpTable.NULL_EXPR, Expressions.box( - Expressions.unbox(operand, toBox), + IgniteExpressions.unboxChecked(operand, fromBox, toBox), toBox)); } else if (fromPrimitive != null && toBox != null) { @@ -322,7 +323,7 @@ else if (fromPrimitive != null && toBox != null) { // Convert it first and generate "Byte.valueOf((byte)x)" // Because there is no method "Byte.valueOf(int)" in Byte return Expressions.box( - Expressions.convert_(operand, toBox.primitiveClass), + IgniteExpressions.convertChecked(operand, fromPrimitive, toBox), toBox); } // Convert datetime types to internal storage type: diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteExpressions.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteExpressions.java new file mode 100644 index 00000000000000..895c179ae388d4 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteExpressions.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.exp; + +import java.lang.reflect.Type; +import java.math.BigDecimal; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.ExpressionType; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Primitive; +import org.apache.calcite.runtime.SqlFunctions; +import org.apache.ignite.internal.processors.query.calcite.util.IgniteMath; +import org.jetbrains.annotations.Nullable; + +/** Calcite liq4j expressions customized for Ignite. */ +public class IgniteExpressions { + /** Make binary expression with arithmetic operations override. */ + public static Expression makeBinary(ExpressionType binaryType, Expression left, Expression right) { + switch (binaryType) { + case Add: + return addExact(left, right); + case Subtract: + return subtractExact(left, right); + case Multiply: + return multiplyExact(left, right); + case Divide: + return divideExact(left, right); + default: + return Expressions.makeBinary(binaryType, left, right); + } + } + + /** Make unary expression with arithmetic operations override. */ + public static Expression makeUnary(ExpressionType unaryType, Expression operand) { + switch (unaryType) { + case Negate: + case NegateChecked: + return negateExact(unaryType, operand); + default: + return Expressions.makeUnary(unaryType, operand); + } + } + + /** Generate expression for method IgniteMath.addExact() for integer subtypes. */ + public static Expression addExact(Expression left, Expression right) { + if (larger(left.getType(), right.getType()).isFixedNumeric()) + return Expressions.call(IgniteMath.class, "addExact", left, right); + + return Expressions.makeBinary(ExpressionType.Add, left, right); + } + + /** Generate expression for method IgniteMath.subtractExact() for integer subtypes. */ + public static Expression subtractExact(Expression left, Expression right) { + if (larger(left.getType(), right.getType()).isFixedNumeric()) + return Expressions.call(IgniteMath.class, "subtractExact", left, right); + + return Expressions.makeBinary(ExpressionType.Subtract, left, right); + } + + /** Generate expression for method IgniteMath.multiplyExact() for integer subtypes. */ + public static Expression multiplyExact(Expression left, Expression right) { + if (larger(left.getType(), right.getType()).isFixedNumeric()) + return Expressions.call(IgniteMath.class, "multiplyExact", left, right); + + return Expressions.makeBinary(ExpressionType.Multiply, left, right); + } + + /** Generate expression for method IgniteMath.divideExact() for integer subtypes. */ + public static Expression divideExact(Expression left, Expression right) { + if (larger(left.getType(), right.getType()).isFixedNumeric()) + return Expressions.call(IgniteMath.class, "divideExact", left, right); + + return Expressions.makeBinary(ExpressionType.Divide, left, right); + } + + /** Generate expression for method IgniteMath.negateExact() for integer subtypes. */ + private static Expression negateExact(ExpressionType unaryType, Expression operand) { + assert unaryType == ExpressionType.Negate || unaryType == ExpressionType.NegateChecked; + + Type opType = operand.getType(); + + if (opType == Integer.TYPE || opType == Long.TYPE || opType == Short.TYPE || opType == Byte.TYPE) + return Expressions.call(IgniteMath.class, "negateExact", operand); + + return Expressions.makeUnary(unaryType, operand); + } + + /** Generate expression for conversion from numeric primitive to numeric primitive with bounds check. */ + public static Expression convertChecked(Expression exp, Primitive fromPrimitive, Primitive toPrimitive) { + if (fromPrimitive.ordinal() <= toPrimitive.ordinal() || !toPrimitive.isFixedNumeric()) + return Expressions.convert_(exp, toPrimitive.primitiveClass); + + return Expressions.call(IgniteMath.class, "convertTo" + + SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", exp); + } + + /** Generate expression for conversion from string to numeric primitive with bounds check. */ + public static Expression parseStringChecked(Expression exp, Primitive toPrimitive) { + return Expressions.call(IgniteMath.class, "convertTo" + + SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", Expressions.new_(BigDecimal.class, exp)); + } + + /** Generate expression for conversion from Number to numeric primitive with bounds check. */ + public static Expression unboxChecked(Expression exp, @Nullable Primitive fromBox, Primitive toPrimitive) { + if ((fromBox != null && fromBox.ordinal() <= toPrimitive.ordinal()) || !toPrimitive.isFixedNumeric()) + return Expressions.unbox(exp, toPrimitive); + + return Expressions.call(IgniteMath.class, "convertTo" + + SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", exp); + } + + /** Find larger in type hierarchy. */ + private static Primitive larger(Type type0, Type type1) { + Primitive primitive0 = Primitive.ofBoxOr(type0); + Primitive primitive1 = Primitive.ofBoxOr(type1); + + return primitive0.ordinal() > primitive1.ordinal() ? primitive0 : primitive1; + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteRexBuilder.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteRexBuilder.java index 7d250c3fc7d0b2..e26d249437cc9e 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteRexBuilder.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteRexBuilder.java @@ -24,6 +24,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.calcite.util.TypeUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -36,8 +37,21 @@ public IgniteRexBuilder(RelDataTypeFactory typeFactory) { /** {@inheritDoc} */ @Override protected RexLiteral makeLiteral(@Nullable Comparable o, RelDataType type, SqlTypeName typeName) { - if (o != null && typeName == SqlTypeName.DECIMAL && TypeUtils.hasScale(type)) - return super.makeLiteral(((BigDecimal)o).setScale(type.getScale(), RoundingMode.HALF_UP), type, typeName); + if (o != null && typeName == SqlTypeName.DECIMAL) { + BigDecimal bd = (BigDecimal)o; + + if (type.getSqlTypeName() == SqlTypeName.BIGINT) { + try { + bd.longValueExact(); + } + catch (ArithmeticException e) { + throw new IgniteSQLException(SqlTypeName.BIGINT.getName() + " overflow", e); + } + } + + if (TypeUtils.hasScale(type)) + return super.makeLiteral(bd.setScale(type.getScale(), RoundingMode.HALF_UP), type, typeName); + } return super.makeLiteral(o, type, typeName); } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/RexImpTable.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/RexImpTable.java index 05f085ce273374..e52342a4320272 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/RexImpTable.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/RexImpTable.java @@ -1244,7 +1244,7 @@ private static class BinaryImplementor extends AbstractRexCallImplementor { argValueList); } } - return Expressions.makeBinary(expressionType, + return IgniteExpressions.makeBinary(expressionType, argValueList.get(0), argValueList.get(1)); } @@ -1312,7 +1312,7 @@ private static class UnaryImplementor extends AbstractRexCallImplementor { && null != backupMethodName) e = Expressions.call(argVal, backupMethodName); else - e = Expressions.makeUnary(expressionType, argVal); + e = IgniteExpressions.makeUnary(expressionType, argVal); if (e.type.equals(argVal.type)) return e; @@ -1771,10 +1771,10 @@ private static class DatetimeArithmeticImplementor case INTERVAL_MINUTE: case INTERVAL_MINUTE_SECOND: case INTERVAL_SECOND: - trop1 = Expressions.convert_( + trop1 = IgniteExpressions.convertChecked( Expressions.divide(trop1, Expressions.constant(DateTimeUtils.MILLIS_PER_DAY)), - int.class); + Primitive.of(long.class), Primitive.of(int.class)); } } break; @@ -1813,9 +1813,9 @@ private static class DatetimeArithmeticImplementor case INTERVAL_SECOND: switch (call.getKind()) { case MINUS: - return normalize(typeName, Expressions.subtract(trop0, trop1)); + return normalize(typeName, IgniteExpressions.subtractExact(trop0, trop1)); default: - return normalize(typeName, Expressions.add(trop0, trop1)); + return normalize(typeName, IgniteExpressions.addExact(trop0, trop1)); } default: diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/RootNode.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/RootNode.java index 5449e19eb38e72..891cff97c79bd0 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/RootNode.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/RootNode.java @@ -327,6 +327,6 @@ private void checkException() { if (e instanceof IgniteSQLException) throw (IgniteSQLException)e; else - throw new IgniteSQLException("An error occurred while query executing.", IgniteQueryErrorCode.UNKNOWN, e); + throw new IgniteSQLException("An error occurred while query executing - " + e.getMessage(), IgniteQueryErrorCode.UNKNOWN, e); } } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 76345aa6895c1d..2565a7bdfd1d11 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -43,6 +43,7 @@ import org.apache.calcite.sql.SqlMerge; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlNumericLiteral; import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlUpdate; @@ -67,6 +68,7 @@ import org.apache.ignite.internal.processors.query.calcite.schema.CacheTableDescriptor; import org.apache.ignite.internal.processors.query.calcite.schema.IgniteCacheTable; import org.apache.ignite.internal.processors.query.calcite.schema.IgniteTable; +import org.apache.ignite.internal.processors.query.calcite.sql.IgniteSqlDecimalLiteral; import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory; import org.apache.ignite.internal.processors.query.calcite.util.IgniteResource; import org.apache.ignite.internal.util.typedef.F; @@ -575,4 +577,22 @@ else if (operandTypeChecker instanceof FamilyOperandTypeChecker) { else super.inferUnknownTypes(inferredType, scope, node); } + + /** {@inheritDoc} */ + @Override public SqlLiteral resolveLiteral(SqlLiteral literal) { + if (literal instanceof SqlNumericLiteral && literal.createSqlType(typeFactory).getSqlTypeName() == SqlTypeName.BIGINT) { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + + if (bd.scale() == 0) { + try { + bd.longValueExact(); + } + catch (ArithmeticException e) { + return new IgniteSqlDecimalLiteral((SqlNumericLiteral)literal); + } + } + } + + return super.resolveLiteral(literal); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionDuplicateKeyException.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/sql/IgniteSqlDecimalLiteral.java similarity index 52% rename from modules/core/src/main/java/org/apache/ignite/transactions/TransactionDuplicateKeyException.java rename to modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/sql/IgniteSqlDecimalLiteral.java index 55ee86734a6d07..20a010e38db44a 100644 --- a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionDuplicateKeyException.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/sql/IgniteSqlDecimalLiteral.java @@ -14,32 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.ignite.internal.processors.query.calcite.sql; -package org.apache.ignite.transactions; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlNumericLiteral; +import org.apache.calcite.sql.type.SqlTypeName; /** - * Exception thrown whenever transaction tries to inserts entry with same mvcc version more than once. + * Class for numeric literal with exact value out of valid long type range. */ -public class TransactionDuplicateKeyException extends TransactionException { +public class IgniteSqlDecimalLiteral extends SqlNumericLiteral { /** */ - private static final long serialVersionUID = 0L; - - /** - * Creates new duplicate ket exception with given error message. - * - * @param msg Error message.\ - * @param cause Optional nested exception (can be {@code null}). - */ - public TransactionDuplicateKeyException(String msg, Exception cause) { - super(msg, cause); + public IgniteSqlDecimalLiteral(SqlNumericLiteral lit) { + super(lit.bigDecimalValue(), lit.getPrec(), lit.getScale(), lit.isExact(), lit.getParserPosition()); } - /** - * Creates new duplicate ket exception with given error message. - * - * @param msg Error message. - */ - public TransactionDuplicateKeyException(String msg) { - super(msg); + /** {@inheritDoc} */ + @Override public RelDataType createSqlType(RelDataTypeFactory typeFactory) { + return typeFactory.createSqlType(SqlTypeName.DECIMAL, bigDecimalValue().precision()); } } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteMath.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteMath.java new file mode 100644 index 00000000000000..f5350d1c18c156 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteMath.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.util; + +import java.math.BigDecimal; +import org.apache.calcite.sql.type.SqlTypeName; + +import static org.apache.calcite.sql.type.SqlTypeName.BIGINT; +import static org.apache.calcite.sql.type.SqlTypeName.INTEGER; +import static org.apache.calcite.sql.type.SqlTypeName.SMALLINT; +import static org.apache.calcite.sql.type.SqlTypeName.TINYINT; + +/** Math operations with overflow checking. */ +public class IgniteMath { + /** */ + private static final BigDecimal UPPER_LONG_BIG_DECIMAL = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE); + + /** */ + private static final BigDecimal LOWER_LONG_BIG_DECIMAL = BigDecimal.valueOf(Long.MIN_VALUE).subtract(BigDecimal.ONE); + + /** */ + private static final Double UPPER_LONG_DOUBLE = (double)Long.MAX_VALUE; + + /** */ + private static final Double LOWER_LONG_DOUBLE = (double)Long.MIN_VALUE; + + /** */ + private static final Float UPPER_LONG_FLOAT = (float)Long.MAX_VALUE; + + /** */ + private static final Float LOWER_LONG_FLOAT = (float)Long.MIN_VALUE; + + /** Returns the sum of its arguments, throwing an exception if the result overflows an {@code long}. */ + public static long addExact(long x, long y) { + long r = x + y; + + if (((x ^ r) & (y ^ r)) < 0) + throw new ArithmeticException(BIGINT.getName() + " overflow"); + + return r; + } + + /** Returns the sum of its arguments, throwing an exception if the result overflows an {@code int}. */ + public static int addExact(int x, int y) { + int r = x + y; + + if (((x ^ r) & (y ^ r)) < 0) + throw new ArithmeticException(INTEGER.getName() + " overflow"); + + return r; + } + + /** Returns the sum of its arguments, throwing an exception if the result overflows an {@code short}. */ + public static short addExact(short x, short y) { + int r = x + y; + + if (r != (short)r) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return (short)r; + } + + /** Returns the sum of its arguments, throwing an exception if the result overflows an {@code byte}. */ + public static byte addExact(byte x, byte y) { + int r = x + y; + + if (r != (byte)r) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return (byte)r; + } + + /** Returns the negation of the argument, throwing an exception if the result overflows an {@code long}. */ + public static long negateExact(long x) { + long res = -x; + + if (x != 0 && x == res) + throw new ArithmeticException(BIGINT.getName() + " overflow"); + + return res; + } + + /** Returns the negation of the argument, throwing an exception if the result overflows an {@code int}. */ + public static int negateExact(int x) { + int res = -x; + + if (x != 0 && x == res) + throw new ArithmeticException(INTEGER.getName() + " overflow"); + + return res; + } + + /** Returns the negation of the argument, throwing an exception if the result overflows an {@code short}. */ + public static short negateExact(short x) { + int res = -x; + + if (res > Short.MAX_VALUE) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return (short)res; + } + + /** Returns the negation of the argument, throwing an exception if the result overflows an {@code byte}. */ + public static byte negateExact(byte x) { + int res = -x; + + if (res > Byte.MAX_VALUE) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return (byte)res; + } + + /** Returns the difference of the arguments, throwing an exception if the result overflows an {@code long}.*/ + public static long subtractExact(long x, long y) { + long r = x - y; + + if (((x ^ y) & (x ^ r)) < 0) + throw new ArithmeticException(BIGINT.getName() + " overflow"); + + return r; + } + + /** Returns the difference of the arguments, throwing an exception if the result overflows an {@code int}.*/ + public static int subtractExact(int x, int y) { + int r = x - y; + + if (((x ^ y) & (x ^ r)) < 0) + throw new ArithmeticException(INTEGER.getName() + " overflow"); + + return r; + } + + /** Returns the difference of the arguments, throwing an exception if the result overflows an {@code short}.*/ + public static short subtractExact(short x, short y) { + int r = x - y; + + if (r != (short)r) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return (short)r; + } + + /** Returns the difference of the arguments, throwing an exception if the result overflows an {@code byte}.*/ + public static byte subtractExact(byte x, byte y) { + int r = x - y; + + if (r != (byte)r) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return (byte)r; + } + + /** Returns the product of the arguments, throwing an exception if the result overflows an {@code long}. */ + public static long multiplyExact(long x, long y) { + long r = x * y; + long ax = Math.abs(x); + long ay = Math.abs(y); + + if ((ax | ay) >>> 31 != 0 && ((y != 0 && r / y != x) || (x == Long.MIN_VALUE && y == -1))) + throw new ArithmeticException(BIGINT.getName() + " overflow"); + + return r; + } + + /** Returns the product of the arguments, throwing an exception if the result overflows an {@code int}. */ + public static int multiplyExact(int x, int y) { + long r = (long)x * (long)y; + + return convertToIntExact(r); + } + + /** Returns the product of the arguments, throwing an exception if the result overflows an {@code short}. */ + public static short multiplyExact(short x, short y) { + int r = x * y; + + if (r != (short)r) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return (short)r; + } + + /** Returns the product of the arguments, throwing an exception if the result overflows an {@code byte}. */ + public static byte multiplyExact(byte x, byte y) { + int r = x * y; + + if (r != (byte)r) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return (byte)r; + } + + /** Returns the quotient of the arguments, throwing an exception if the result overflows an {@code long}. */ + public static long divideExact(long x, long y) { + if (y == -1) + return negateExact(x); + + return x / y; + } + + /** Returns the quotient of the arguments, throwing an exception if the result overflows an {@code int}. */ + public static int divideExact(int x, int y) { + if (y == -1) + return negateExact(x); + + return x / y; + } + + /** Returns the quotient of the arguments, throwing an exception if the result overflows an {@code short}. */ + public static short divideExact(short x, short y) { + if (y == -1) + return negateExact(x); + + return (short)(x / y); + } + + /** Returns the quotient of the arguments, throwing an exception if the result overflows an {@code byte}. */ + public static byte divideExact(byte x, byte y) { + if (y == -1) + return negateExact(x); + + return (byte)(x / y); + } + + /** */ + private static void checkNumberLongBounds(SqlTypeName type, Number x) { + if (x instanceof BigDecimal) { + if ((((BigDecimal)x).compareTo(UPPER_LONG_BIG_DECIMAL) < 0 && ((BigDecimal)x).compareTo(LOWER_LONG_BIG_DECIMAL) > 0)) + return; + } + else if (x instanceof Double) { + if ((((Double)x).compareTo(UPPER_LONG_DOUBLE) <= 0 && ((Double)x).compareTo(LOWER_LONG_DOUBLE) >= 0)) + return; + } + else if (x instanceof Float) { + if ((((Float)x).compareTo(UPPER_LONG_FLOAT) <= 0 && ((Float)x).compareTo(LOWER_LONG_FLOAT) >= 0)) + return; + } + else + return; + + throw new ArithmeticException(type.getName() + " overflow"); + } + + /** Cast value to {@code long}, throwing an exception if the result overflows an {@code long}. */ + public static long convertToLongExact(Number x) { + checkNumberLongBounds(BIGINT, x); + + return x.longValue(); + } + + /** Cast value to {@code long}, throwing an exception if the result overflows an {@code long}. */ + public static long convertToLongExact(double x) { + if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) + throw new ArithmeticException(BIGINT.getName() + " overflow"); + + return (long)x; + } + + /** Cast value to {@code int}, throwing an exception if the result overflows an {@code int}. */ + public static int convertToIntExact(long x) { + int res = (int)x; + + if (res != x) + throw new ArithmeticException(INTEGER.getName() + " overflow"); + + return res; + } + + /** Cast value to {@code int}, throwing an exception if the result overflows an {@code int}. */ + public static int convertToIntExact(double x) { + if (x > Integer.MAX_VALUE || x < Integer.MIN_VALUE) + throw new ArithmeticException(INTEGER.getName() + " overflow"); + + return (int)x; + } + + /** Cast value to {@code int}, throwing an exception if the result overflows an {@code int}. */ + public static int convertToIntExact(Number x) { + checkNumberLongBounds(INTEGER, x); + + return convertToIntExact(x.longValue()); + } + + /** Cast value to {@code short}, throwing an exception if the result overflows an {@code short}. */ + public static short convertToShortExact(long x) { + short res = (short)x; + + if (res != x) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return res; + } + + /** Cast value to {@code short}, throwing an exception if the result overflows an {@code short}. */ + public static short convertToShortExact(double x) { + if (x > Short.MAX_VALUE || x < Short.MIN_VALUE) + throw new ArithmeticException(SMALLINT.getName() + " overflow"); + + return (short)x; + } + + /** Cast value to {@code short}, throwing an exception if the result overflows an {@code short}. */ + public static short convertToShortExact(Number x) { + checkNumberLongBounds(SMALLINT, x); + + return convertToShortExact(x.longValue()); + } + + /** Cast value to {@code byte}, throwing an exception if the result overflows an {@code byte}. */ + public static byte convertToByteExact(long x) { + byte res = (byte)x; + + if (res != x) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return res; + } + + /** Cast value to {@code byte}, throwing an exception if the result overflows an {@code byte}. */ + public static byte convertToByteExact(double x) { + if (x > Byte.MAX_VALUE || x < Byte.MIN_VALUE) + throw new ArithmeticException(TINYINT.getName() + " overflow"); + + return (byte)x; + } + + /** Cast value to {@code byte}, throwing an exception if the result overflows an {@code byte}. */ + public static byte convertToByteExact(Number x) { + checkNumberLongBounds(TINYINT, x); + + return convertToByteExact(x.longValue()); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DataTypesTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DataTypesTest.java index a3cf6362f24266..b6dd0aa7b82af2 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DataTypesTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DataTypesTest.java @@ -466,4 +466,139 @@ public void testNumericConversion() { .returns((byte)7, (short)7, 7, 7L, BigDecimal.valueOf(7), 7f, 7d) .check(); } + + /** */ + @Test + public void testArithmeticOverflow() { + // BIGINT + assertThrows("select CAST(9223372036854775807.5 + 1 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select 9223372036854775807 + 1", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select 9223372036854775807 * 2", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select -9223372036854775808 - 1", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select -(-9223372036854775807 - 1)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select -CAST(-9223372036854775808 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("select -(?)", IgniteSQLException.class, "BIGINT overflow", -9223372036854775808L); + assertThrows("select -9223372036854775808/-1", IgniteSQLException.class, "BIGINT overflow"); + + // INTEGER + assertThrows("select CAST(CAST(3000000000.0 + 1 AS DOUBLE) AS INTEGER)", + IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select CAST(9223372036854775807.5 + 9223372036854775807.5 AS INTEGER)", + IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select CAST(2147483647.5 + 1 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select 2147483647 + 1", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select 2147483647 * 2", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select -2147483648 - 1", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select -(-2147483647 - 1)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select -CAST(-2147483648 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("select -(?)", IgniteSQLException.class, "INTEGER overflow", -2147483648); + assertThrows("select -2147483648/-1", IgniteSQLException.class, "INTEGER overflow"); + + // SMALLINT + assertThrows("select CAST(CAST(90000.0 + 1 AS FLOAT) AS SMALLINT)", + IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select CAST(9223372036854775807.5 + 9223372036854775807.5 AS SMALLINT)", + IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select 32000::smallint + 1000::smallint", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select 17000::smallint * 2::smallint", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select -32000::smallint - 1000::smallint", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select -(-32767::smallint - 1::smallint)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select -CAST(-32768 AS smallint)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("select -CAST(? AS smallint)", IgniteSQLException.class, "SMALLINT overflow", -32768); + assertThrows("select -32768::smallint/-1::smallint", IgniteSQLException.class, "SMALLINT overflow"); + + // TINYINT + assertThrows("select CAST(CAST(200.0 + 1 AS FLOAT) AS TINYINT)", + IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select CAST(9223372036854775807.5 + 9223372036854775807.5 AS TINYINT)", + IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select 2::tinyint + 127::tinyint", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select 2::tinyint * 127::tinyint", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select -2::tinyint - 127::tinyint", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select -(-127::tinyint - 1::tinyint)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select -CAST(-128 AS tinyint)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("select -CAST(? AS tinyint)", IgniteSQLException.class, "TINYINT overflow", -128); + assertThrows("select -128::tinyint/-1::tinyint", IgniteSQLException.class, "TINYINT overflow"); + } + + /** */ + @Test + public void testCastDecimalOverflows() { + // BIGINT + assertQuery("SELECT CAST(9223372036854775807.1 AS BIGINT)").returns(9223372036854775807L).check(); + assertQuery("SELECT CAST(9223372036854775807.9 AS BIGINT)").returns(9223372036854775807L).check(); + assertQuery("SELECT CAST(9223372036854775808.9 - 1 AS BIGINT)").returns(9223372036854775807L).check(); + assertThrows("SELECT CAST(9223372036854775808 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST(9223372036854775808.1 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST(-9223372036854775809 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST(-9223372036854775809.1 AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertQuery("SELECT CAST(-9223372036854775808.1 AS BIGINT)").returns(-9223372036854775808L).check(); + assertQuery("SELECT CAST(-9223372036854775808.9 AS BIGINT)").returns(-9223372036854775808L).check(); + assertQuery("SELECT CAST(-9223372036854775809.9 + 1 AS BIGINT)").returns(-9223372036854775808L).check(); + assertQuery("SELECT CAST('9223372036854775807.1' AS BIGINT)").returns(9223372036854775807L).check(); + assertQuery("SELECT CAST('9223372036854775807.9' AS BIGINT)").returns(9223372036854775807L).check(); + assertThrows("SELECT CAST('9223372036854775808' AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST('9223372036854775808.1' AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST('-9223372036854775809' AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertThrows("SELECT CAST('-9223372036854775809.1' AS BIGINT)", IgniteSQLException.class, "BIGINT overflow"); + assertQuery("SELECT CAST('-9223372036854775808.1' AS BIGINT)").returns(-9223372036854775808L).check(); + assertQuery("SELECT CAST('-9223372036854775808.9' AS BIGINT)").returns(-9223372036854775808L).check(); + + // INTEGER + assertQuery("SELECT CAST(2147483647.1 AS INTEGER)").returns(2147483647).check(); + assertQuery("SELECT CAST(2147483647.9 AS INTEGER)").returns(2147483647).check(); + assertQuery("SELECT CAST(2147483648.9 - 1 AS INTEGER)").returns(2147483647).check(); + assertThrows("SELECT CAST(2147483648 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST(2147483648.1 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST(-2147483649 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST(-2147483649.1 AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertQuery("SELECT CAST(-2147483648.1 AS INTEGER)").returns(-2147483648).check(); + assertQuery("SELECT CAST(-2147483648.9 AS INTEGER)").returns(-2147483648).check(); + assertQuery("SELECT CAST('2147483647.1' AS INTEGER)").returns(2147483647).check(); + assertQuery("SELECT CAST('2147483647.9' AS INTEGER)").returns(2147483647).check(); + assertThrows("SELECT CAST('2147483648' AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST('2147483648.1' AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST('-2147483649' AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT CAST('-2147483649.1' AS INTEGER)", IgniteSQLException.class, "INTEGER overflow"); + assertQuery("SELECT CAST('-2147483648.1' AS INTEGER)").returns(-2147483648).check(); + assertQuery("SELECT CAST('-2147483648.9' AS INTEGER)").returns(-2147483648).check(); + + // SMALLINT + assertQuery("SELECT CAST(32767.1 AS SMALLINT)").returns((short)32767).check(); + assertQuery("SELECT CAST(32767.9 AS SMALLINT)").returns((short)32767).check(); + assertQuery("SELECT CAST(32768.9 - 1 AS SMALLINT)").returns((short)32767).check(); + assertThrows("SELECT CAST(32768 AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST(32768.1 AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST(-32769 AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST(-32769.1 AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertQuery("SELECT CAST(-32768.1 AS SMALLINT)").returns((short)-32768).check(); + assertQuery("SELECT CAST(-32768.9 AS SMALLINT)").returns((short)-32768).check(); + assertQuery("SELECT CAST('32767.1' AS SMALLINT)").returns((short)32767).check(); + assertQuery("SELECT CAST('32767.9' AS SMALLINT)").returns((short)32767).check(); + assertThrows("SELECT CAST('32768' AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST('32768.1' AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST('-32769' AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertThrows("SELECT CAST('-32769.1' AS SMALLINT)", IgniteSQLException.class, "SMALLINT overflow"); + assertQuery("SELECT CAST('-32768.1' AS SMALLINT)").returns((short)-32768).check(); + assertQuery("SELECT CAST('-32768.9' AS SMALLINT)").returns((short)-32768).check(); + + // TINYINT + assertQuery("SELECT CAST(127.1 AS TINYINT)").returns((byte)127).check(); + assertQuery("SELECT CAST(127.9 AS TINYINT)").returns((byte)127).check(); + assertQuery("SELECT CAST(128.9 - 1 AS TINYINT)").returns((byte)127).check(); + assertThrows("SELECT CAST(128 AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST(128.1 AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST(-129 AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST(-129.1 AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertQuery("SELECT CAST(-128.1 AS TINYINT)").returns((byte)-128).check(); + assertQuery("SELECT CAST(-128.9 AS TINYINT)").returns((byte)-128).check(); + assertQuery("SELECT CAST('127.1' AS TINYINT)").returns((byte)127).check(); + assertQuery("SELECT CAST('127.9' AS TINYINT)").returns((byte)127).check(); + assertThrows("SELECT CAST('128' AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST('128.1' AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST('-129' AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertThrows("SELECT CAST('-129.1' AS TINYINT)", IgniteSQLException.class, "TINYINT overflow"); + assertQuery("SELECT CAST('-128.1' AS TINYINT)").returns((byte)-128).check(); + assertQuery("SELECT CAST('-128.9' AS TINYINT)").returns((byte)-128).check(); + } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/IntervalTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/IntervalTest.java index f3e94a95330603..1d03d5f36337ed 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/IntervalTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/IntervalTest.java @@ -331,6 +331,20 @@ public void testIntervalArithmetic() { assertEquals(Duration.ofSeconds(31), eval("INTERVAL '1:2' MINUTE TO SECOND / 2")); assertEquals(Duration.ofSeconds(1862), eval("INTERVAL '1:2:4' HOUR TO SECOND / 2")); assertEquals(Duration.ofMillis(1862228), eval("INTERVAL '0 1:2:4.456' DAY TO SECOND / 2")); + + // Interval range overflow + assertThrows("SELECT INTERVAL 5000000 MONTHS * 1000", + IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT DATE '2021-01-01' + INTERVAL 999999999999 DAY", + IgniteSQLException.class, "BIGINT overflow"); // Overflow for interval type (long). + assertThrows("SELECT DATE '2021-01-01' + INTERVAL 3000000000 DAYS", + IgniteSQLException.class, "INTEGER overflow"); // Overflow for date type (integer). + assertThrows("SELECT DATE '2021-01-01' + INTERVAL -999999999 YEAR", + IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT INTERVAL 1000000000 YEARS + INTERVAL 1 MONTH", + IgniteSQLException.class, "INTEGER overflow"); + assertThrows("SELECT INTERVAL 100000000000 DAYS + INTERVAL 100000000000 DAYS", + IgniteSQLException.class, "BIGINT overflow"); } /** diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TableDmlIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TableDmlIntegrationTest.java index 513db94d7c08eb..0e1e1435c83e88 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TableDmlIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TableDmlIntegrationTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.binary.BinaryObjectBuilder; @@ -585,6 +586,54 @@ public void testInsertDuplicateKey() { "Failed to INSERT some keys because they are already in cache"); } + /** */ + @Test + public void testInsertValueOverflow() { + List> args = F.asList( + F.asList(SqlTypeName.BIGINT.getName(), Long.MAX_VALUE, Long.MIN_VALUE), + F.asList(SqlTypeName.INTEGER.getName(), (long)Integer.MAX_VALUE, (long)Integer.MIN_VALUE), + F.asList(SqlTypeName.SMALLINT.getName(), (long)Short.MAX_VALUE, (long)Short.MIN_VALUE), + F.asList(SqlTypeName.TINYINT.getName(), (long)Byte.MAX_VALUE, (long)Byte.MIN_VALUE) + ); + + for (List arg : args) { + try { + String type = (String)arg.get(0); + long max = (Long)arg.get(1); + long min = (Long)arg.get(2); + + sql(String.format("CREATE TABLE TEST_SOURCE (ID INT PRIMARY KEY, VAL %s)", type)); + sql(String.format("CREATE TABLE TEST_DEST (ID INT PRIMARY KEY, VAL %s)", type)); + + sql("INSERT INTO TEST_SOURCE VALUES (1, 1)"); + sql(String.format("INSERT INTO TEST_SOURCE VALUES (2, %d)", max)); + sql("INSERT INTO TEST_SOURCE VALUES (3, -1)"); + sql(String.format("INSERT INTO TEST_SOURCE VALUES (4, %d)", min)); + + BigDecimal moreThanMax = new BigDecimal(max).add(BigDecimal.ONE); + + assertThrows(String.format("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, %s)", moreThanMax.toString()), + IgniteSQLException.class, type + " overflow"); + assertThrows(String.format("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, %d + 1)", max), + IgniteSQLException.class, type + " overflow"); + assertThrows(String.format("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, %d - 1)", min), + IgniteSQLException.class, type + " overflow"); + assertThrows(String.format("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, %d + (SELECT 1))", max), + IgniteSQLException.class, type + " overflow"); + assertThrows(String.format("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, %d + (SELECT -1))", min), + IgniteSQLException.class, type + " overflow"); + assertThrows("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, (SELECT SUM(VAL) FROM TEST_SOURCE WHERE VAL > 0))", + IgniteSQLException.class, type + " overflow"); + assertThrows("INSERT INTO TEST_DEST (ID, VAL) VALUES (1, (SELECT SUM(VAL) FROM TEST_SOURCE WHERE VAL < 0))", + IgniteSQLException.class, type + " overflow"); + } + finally { + sql("DROP TABLE TEST_SOURCE"); + sql("DROP TABLE TEST_DEST"); + } + } + } + /** */ private void checkDefaultValue(String sqlType, String sqlVal, Object expectedVal) { try { diff --git a/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java b/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java index de0b11b7ead64c..81b17713c2c7d5 100644 --- a/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java @@ -42,8 +42,8 @@ import org.apache.ignite.events.JobEvent; import org.apache.ignite.events.TaskEvent; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; import org.apache.ignite.internal.processors.rest.GridRestCommand; -import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; import org.apache.ignite.internal.processors.security.AbstractSecurityTest; import org.apache.ignite.internal.processors.security.impl.TestSecurityData; import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider; @@ -64,7 +64,7 @@ public abstract class AbstractEventSecurityContextTest extends AbstractSecurityT protected static final Map> LISTENED_EVTS = new ConcurrentHashMap<>(); /** Custom object mapper for HTTP REST API. */ - private static final ObjectMapper OBJECT_MAPPER = new GridJettyObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new IgniteObjectMapper(); /** Port for REST client connection. */ private static final String DFLT_REST_PORT = "11080"; diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java index 5ee18354c9163b..6b39ccb587d5b7 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.concurrent.Callable; import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.internal.util.lang.RunnableX; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; @@ -154,18 +155,35 @@ public void testExecute() throws SQLException { } /** - * + * Checks whether it's impossible to insert duplicate in single key statement. */ @Test - public void testDuplicateKeys() { - jcache(0).put("p2", new Person(2, "Joe", "Black", 35)); + public void testDuplicateSingleKey() { + doTestDuplicate( + () -> stmt.execute(SQL), + "insert into Person(_key, id, firstName, lastName, age, data) values " + + "('p2', 2, 'Joe', 'Black', 35, RAWTOHEX('Black'))" + ); + } - Throwable reason = GridTestUtils.assertThrows(log, new Callable() { - /** {@inheritDoc} */ - @Override public Object call() throws Exception { - return stmt.execute(SQL); - } - }, SQLException.class, null); + /** + * Checks whether it's impossible to insert duplicate in multiple keys statement. + */ + @Test + public void testDuplicateMultipleKeys() { + doTestDuplicate( + () -> jcache(0).put("p2", new Person(2, "Joe", "Black", 35)), + SQL + ); + } + + /** + * + */ + private void doTestDuplicate(RunnableX initClosure, String sql) { + initClosure.run(); + + Throwable reason = GridTestUtils.assertThrows(log, () -> stmt.execute(sql), SQLException.class, null); reason = reason.getCause(); diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java index 9976c1c52ca12e..e38386d8bdce43 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java @@ -29,7 +29,7 @@ import java.util.Map; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; import org.apache.ignite.internal.util.typedef.internal.SB; import static org.apache.ignite.IgniteSystemProperties.IGNITE_JETTY_PORT; @@ -48,7 +48,7 @@ public abstract class JettyRestProcessorCommonSelfTest extends AbstractRestProce private static final int DFLT_REST_PORT = 8091; /** JSON to java mapper. */ - protected static final ObjectMapper JSON_MAPPER = new GridJettyObjectMapper(); + protected static final ObjectMapper JSON_MAPPER = new IgniteObjectMapper(); /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java index 44f3ddad44f50c..c9f46abc2b3d13 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java @@ -23,8 +23,8 @@ import java.sql.Statement; import java.util.Arrays; import java.util.HashSet; -import java.util.concurrent.Callable; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.lang.RunnableX; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.ListeningTestLogger; import org.apache.ignite.testframework.LogListener; @@ -190,11 +190,33 @@ public void testPreparedExecute() throws SQLException { } /** - * + * Checks whether it's impossible to insert duplicate in single key statement. + */ + @Test + public void testDuplicateSingleKey() throws InterruptedException { + doTestDuplicate( + () -> stmt.execute(SQL), + "insert into Person(_key, id, firstName, lastName, age) values " + + "('p2', 2, 'Joe', 'Black', 35)" + ); + } + + /** + * Checks whether it's impossible to insert duplicate in multiple keys statement. */ @Test - public void testDuplicateKeys() throws InterruptedException { - jcache(0).put("p2", new Person(2, "Joe", "Black", 35)); + public void testDuplicateMultipleKeys() throws InterruptedException { + doTestDuplicate( + () -> jcache(0).put("p2", new Person(2, "Joe", "Black", 35)), + SQL + ); + } + + /** + * + */ + private void doTestDuplicate(RunnableX initClosure, String sql) throws InterruptedException { + initClosure.run(); LogListener lsnr = LogListener .matches("Failed to execute SQL query") @@ -202,12 +224,7 @@ public void testDuplicateKeys() throws InterruptedException { srvLog.registerListener(lsnr); - GridTestUtils.assertThrowsAnyCause(log, new Callable() { - /** {@inheritDoc} */ - @Override public Object call() throws Exception { - return stmt.execute(SQL); - } - }, SQLException.class, + GridTestUtils.assertThrowsAnyCause(log, () -> stmt.execute(sql), SQLException.class, "Failed to INSERT some keys because they are already in cache [keys=[p2]]"); assertFalse(lsnr.check(1000L)); diff --git a/modules/control-utility/pom.xml b/modules/control-utility/pom.xml index 56964c5a55a0da..6dd782bf2a20c1 100644 --- a/modules/control-utility/pom.xml +++ b/modules/control-utility/pom.xml @@ -85,6 +85,12 @@ test + + ${project.groupId} + ignite-json + test + + org.apache.logging.log4j log4j-slf4j-impl diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java index f486c27d906d02..1511bf703ab880 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java @@ -32,6 +32,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Scanner; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -51,6 +53,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.difflib.text.DiffRow; import com.github.difflib.text.DiffRowGenerator; import org.apache.ignite.Ignite; @@ -76,6 +80,7 @@ import org.apache.ignite.internal.commandline.ArgumentParser; import org.apache.ignite.internal.commandline.CommandHandler; import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; import org.apache.ignite.internal.management.IgniteCommandRegistry; import org.apache.ignite.internal.management.api.HelpCommand; import org.apache.ignite.internal.management.api.Positional; @@ -84,6 +89,7 @@ import org.apache.ignite.internal.management.cache.CacheDestroyCommand; import org.apache.ignite.internal.management.cache.IdleVerifyDumpTask; import org.apache.ignite.internal.management.cache.scan.DefaultCacheScanTaskFormat; +import org.apache.ignite.internal.management.cache.scan.JsonCacheScanTaskFormat; import org.apache.ignite.internal.management.cache.scan.TableCacheScanTaskFormat; import org.apache.ignite.internal.management.tx.TxTaskResult; import org.apache.ignite.internal.processors.cache.CacheGroupContext; @@ -1699,7 +1705,13 @@ private void dataForScanTest() { c3.put(1, new TestClass2( 1, + new boolean[]{true, false}, + new char[] {'t', 'e', 's', 't'}, + new short[] {1, 2, 3}, new int[]{2, 3}, + new long[] {4, 5}, + new float[]{}, + new double[]{42.0}, Collections.singletonMap("some_key", "some_value"), new String[] {"s1", "s2", "s3"}, DATE, @@ -1709,7 +1721,13 @@ private void dataForScanTest() { c3.put(2, new TestClass2( 2, - new int[]{3, 4}, + new boolean[]{true, false}, + new char[] {'t', 'e', 's', 't'}, + new short[] {1, 2, 3}, + new int[]{2, 3}, + new long[] {4, 5}, + new float[]{123.0f}, + new double[] {0}, Collections.singletonMap("1", "2"), new String[] {"s4", "s5", "s6"}, DATE, @@ -1719,7 +1737,13 @@ private void dataForScanTest() { c3.put(3, new TestClass2( 3, - new int[]{4, 5}, + new boolean[]{true, false}, + new char[] {'t', 'e', 's', 't'}, + new short[] {1, 2, 3}, + new int[]{2, 3}, + new long[] {4, 5}, + new float[]{123.0f}, + new double[] {1}, Collections.singletonMap("xxx", "yyy"), new String[] {"s7", "s8", "s9"}, DATE, @@ -1759,9 +1783,140 @@ public void testCacheScanTableFormat() { assertContains(log, testOut.toString(), "some_key=some_value"); assertContains(log, testOut.toString(), "xxx=yyy"); + assertContains(log, testOut.toString(), "[s1, s2, s3]"); assertContains(log, testOut.toString(), DATE.toString()); } + /** */ + @Test + public void testCacheScanJsonFormat() throws Exception { + injectTestSystemOut(); + + autoConfirmation = false; + + dataForScanTest(); + + assertEquals(EXIT_CODE_OK, execute("--cache", SCAN, "--output-format", JsonCacheScanTaskFormat.NAME, "cache1")); + + Scanner sc = new Scanner(testOut.toString()); + + while (sc.hasNextLine() && !Objects.equals(sc.nextLine().trim(), "data")) ; + + TypeReference> typeRef = new TypeReference>() { + // No-op. + }; + + ObjectMapper mapper = new ObjectMapper(); + + Set keys = new HashSet<>(); + + for (int i = 0; i < 3; i++) { + assertTrue(sc.hasNextLine()); + + Map entryFromJson = mapper.readValue(sc.nextLine(), typeRef); + + int key = (int)((Map)entryFromJson.get("key")).get("id"); + + keys.add(key); + + Map val = (Map)entryFromJson.get("value"); + + if (key == 1) + assertEquals(JOHN, val.get("fio")); + else if (key == 2) + assertEquals(SARAH, val.get("fio")); + else + assertEquals(KYLE, val.get("fio")); + + assertEquals(key + 1, val.get("salary")); + } + + assertTrue(keys.containsAll(Arrays.asList(1, 2, 3))); + + keys.clear(); + + assertEquals(EXIT_CODE_OK, execute("--cache", SCAN, "--output-format", JsonCacheScanTaskFormat.NAME, "cache2")); + + sc = new Scanner(testOut.toString()); + + while (sc.hasNextLine() && !Objects.equals(sc.nextLine().trim(), "data")) ; + + for (int i = 0; i < 3; i++) { + assertTrue(sc.hasNextLine()); + + Map entryFromJson = mapper.readValue(sc.nextLine(), typeRef); + + int key = (int)entryFromJson.get("key"); + + keys.add(key); + + String val = (String)entryFromJson.get("value"); + + if (key == 1) + assertEquals(JOHN, val); + else if (key == 2) + assertEquals(SARAH, val); + else + assertEquals(KYLE, val); + } + + assertTrue(keys.containsAll(Arrays.asList(1, 2, 3))); + + keys.clear(); + + assertEquals(EXIT_CODE_OK, execute("--cache", SCAN, "--output-format", JsonCacheScanTaskFormat.NAME, "cache3")); + + sc = new Scanner(testOut.toString()); + + while (sc.hasNextLine() && !Objects.equals(sc.nextLine().trim(), "data")) ; + + for (int i = 0; i < 3; i++) { + assertTrue(sc.hasNextLine()); + + Map entryFromJson = mapper.readValue(sc.nextLine(), typeRef); + + int key = (int)entryFromJson.get("key"); + + keys.add(key); + + Map val = (Map)entryFromJson.get("value"); + + assertEquals(key, val.get("i")); + assertEquals(Arrays.asList(true, false), val.get("booleans")); + assertEquals("test", val.get("chars")); + assertEquals(Arrays.asList(1, 2, 3), val.get("shorts")); + assertEquals(Arrays.asList(2, 3), val.get("ints")); + assertEquals(Arrays.asList(4, 5), val.get("longs")); + assertEquals(Arrays.asList(1, 2, 3), val.get("list")); + assertEquals(IgniteObjectMapper.DATE_FORMAT.format(DATE), val.get("date")); + + int firstIdx = i * 3 + 1; + + assertEquals(Arrays.asList("s" + firstIdx, "s" + (firstIdx + 1), "s" + (firstIdx + 2)), val.get("strArr")); + + if (key == 1) { + assertTrue(((List)val.get("floats")).isEmpty()); + assertEquals(Arrays.asList(42.0d), val.get("doubles")); + assertEquals(Collections.singletonMap("some_key", "some_value"), val.get("map")); + assertEquals(AccessLevel.USER.toString(), val.get("enm")); + } + else if (key == 2) { + assertEquals(Arrays.asList(123d), val.get("floats")); + assertEquals(Arrays.asList(0d), val.get("doubles")); + assertEquals(Collections.singletonMap("1", "2"), val.get("map")); + assertEquals(AccessLevel.USER.toString(), val.get("enm")); + } + else { + assertEquals(Arrays.asList(123d), val.get("floats")); + assertEquals(Arrays.asList(1d), val.get("doubles")); + assertEquals(Collections.singletonMap("xxx", "yyy"), val.get("map")); + assertEquals(AccessLevel.SUPER.toString(), val.get("enm")); + } + } + + assertTrue(keys.containsAll(Arrays.asList(1, 2, 3))); + } + /** */ @Test public void testCacheScanLimit() { @@ -2384,9 +2539,27 @@ private static class TestClass2 { /** */ private final int i; + /** */ + private final boolean[] booleans; + + /** */ + private final char[] chars; + + /** */ + private final short[] shorts; + /** */ private final int[] ints; + /** */ + private final long[] longs; + + /** */ + private final float[] floats; + + /** */ + private final double[] doubles; + /** */ private final Map map; @@ -2403,9 +2576,29 @@ private static class TestClass2 { private final AccessLevel enm; /** */ - public TestClass2(int i, int[] ints, Map map, String[] strArr, Date date, List list, AccessLevel enm) { + public TestClass2( + int i, + boolean[] booleans, + char[] chars, + short[] shorts, + int[] ints, + long[] longs, + float[] floats, + double[] doubles, + Map map, + String[] strArr, + Date date, + List list, + AccessLevel enm + ) { this.i = i; + this.booleans = booleans; + this.chars = chars; + this.shorts = shorts; this.ints = ints; + this.longs = longs; + this.floats = floats; + this.doubles = doubles; this.map = map; this.strArr = strArr; this.date = date; diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java index 125d9c959fe7ca..b290b319bb1925 100644 --- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java +++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata; import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.Dump; import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.Dump.DumpedPartitionIterator; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpConsumerKernalContextAware; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -80,7 +81,10 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) { try (Dump dump = new Dump(cfg.dumpRoot(), null, cfg.keepBinary(), cfg.keepRaw(), encryptionSpi(), log)) { DumpConsumer cnsmr = cfg.consumer(); - cnsmr.start(); + if (cnsmr instanceof DumpConsumerKernalContextAware) + ((DumpConsumerKernalContextAware)cnsmr).start(dump.context()); + else + cnsmr.start(); try { File[] files = new File(cfg.dumpRoot(), DFLT_MARSHALLER_PATH).listFiles(BinaryUtils::notTmpFile); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/scan/DefaultCacheScanTaskFormat.java b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/scan/DefaultCacheScanTaskFormat.java index f6aa82b0fa81c8..d683cfbf6c9292 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/scan/DefaultCacheScanTaskFormat.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/scan/DefaultCacheScanTaskFormat.java @@ -79,15 +79,42 @@ static String valueOf(Object o) { if (o == null) return "null"; - if (o instanceof byte[]) + if (o instanceof byte[]) { return "size=" + ((byte[])o).length; - - if (o instanceof Byte[]) + } + else if (o instanceof Byte[]) { return "size=" + ((Byte[])o).length; - - if (o instanceof Object[]) { - return "size=" + ((Object[])o).length + - ", values=[" + S.joinToString(Arrays.asList((Object[])o), ", ", "...", 120, 0) + "]"; + } + else if (o instanceof boolean[]) { + boolean[] arr = (boolean[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof char[]) { + char[] arr = (char[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof short[]) { + short[] arr = (short[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof int[]) { + int[] arr = (int[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof long[]) { + long[] arr = (long[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof float[]) { + float[] arr = (float[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof double[]) { + double[] arr = (double[])o; + return arrayValue(arr.length, Arrays.toString(arr)); + } + else if (o instanceof Object[]) { + return arrayValue(((Object[])o).length, "[" + S.joinToString(Arrays.asList((Object[])o), ", ", "...", 120, 0)) + "]"; } if (o instanceof BinaryObject) @@ -96,6 +123,11 @@ static String valueOf(Object o) { return o.toString(); } + /** */ + static String arrayValue(int length, String values) { + return "size=" + length + ", values=" + values; + } + /** * Convert Binary object to string. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java index c26c56f88b164a..7cefa552380338 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java @@ -386,6 +386,11 @@ private File dumpGroupDirectory(String node, int grpId) { return grpDirs[0]; } + /** @return Kernal context. */ + public GridKernalContext context() { + return cctx; + } + /** {@inheritDoc} */ @Override public void close() throws Exception { closeAllComponents(cctx); diff --git a/modules/ducktests/src/main/java/java/net/BlockingDnsInet6AddressImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpConsumerKernalContextAware.java similarity index 59% rename from modules/ducktests/src/main/java/java/net/BlockingDnsInet6AddressImpl.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpConsumerKernalContextAware.java index 47a0a025241584..ec24a4921f8b47 100644 --- a/modules/ducktests/src/main/java/java/net/BlockingDnsInet6AddressImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpConsumerKernalContextAware.java @@ -14,21 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package java.net; -/** */ -public class BlockingDnsInet6AddressImpl extends Inet6AddressImpl { - /** {@inheritDoc} */ - @Override public InetAddress[] lookupAllHostAddr(String hostname) throws UnknownHostException { - DnsBlocker.INSTANCE.onHostResolve(this, hostname); +package org.apache.ignite.internal.processors.cache.persistence.snapshot.dump; - return super.lookupAllHostAddr(hostname); - } +import org.apache.ignite.dump.DumpConsumer; +import org.apache.ignite.internal.GridKernalContext; - /** {@inheritDoc} */ - @Override public String getHostByAddr(byte[] addr) throws UnknownHostException { - DnsBlocker.INSTANCE.onAddrResolve(this, addr); - - return super.getHostByAddr(addr); - } +/** + * Dump consumer that needs to use {@link GridKernalContext} must implement this interface. + */ +public interface DumpConsumerKernalContextAware extends DumpConsumer { + /** + * Starts the consumer with the kernal context provided. + * + * @param ctx Kernal context. + */ + void start(GridKernalContext ctx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java index ef6686253f65b4..48a0aaa2006d05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java @@ -1162,13 +1162,10 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { int cnt = 0; - boolean stop = false; boolean pageSent = false; Collection data = new ArrayList<>(pageSize); - AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion(); - final boolean statsEnabled = cctx.statisticsEnabled(); final boolean readEvt = cctx.events().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); @@ -1289,20 +1286,12 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { onPageReady(loc, qryInfo, res.metadata(), data, finished, null); - pageSent = true; - res.onPageSend(); if (!finished) rmvIter = false; - if (!qryInfo.allPages()) - return; - - data = new ArrayList<>(pageSize); - - if (stop) - break; // while + return; } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/SqlListenerUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/SqlListenerUtils.java index 891496426a542c..de53c276ac5794 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/SqlListenerUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/SqlListenerUtils.java @@ -31,7 +31,6 @@ import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.jetbrains.annotations.Nullable; /** @@ -295,8 +294,6 @@ public static boolean isPlainType(Class cls) { public static int exceptionToSqlErrorCode(Throwable e) { if (e instanceof QueryCancelledException) return IgniteQueryErrorCode.QUERY_CANCELED; - if (e instanceof TransactionDuplicateKeyException) - return IgniteQueryErrorCode.DUPLICATE_KEY; if (e instanceof IgniteSQLException) return ((IgniteSQLException)e).statusCode(); else diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/callback/PlatformCallbackGateway.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/callback/PlatformCallbackGateway.java index 5d6e57d3faedb0..cd54aed9cdf3c9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/callback/PlatformCallbackGateway.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/callback/PlatformCallbackGateway.java @@ -181,12 +181,14 @@ public void cacheInvoke(long memPtr) { * Perform native task map. Do not throw exceptions, serializing them to the output stream instead. * * @param memPtr Memory pointer. + * @param ses Platform compute task session proxy. */ - public void computeTaskMap(long memPtr) { + public void computeTaskMap(long memPtr, PlatformTargetProxy ses) { enter(); try { - PlatformCallbackUtils.inLongOutLong(envPtr, PlatformCallbackOp.ComputeTaskMap, memPtr); + PlatformCallbackUtils.inLongLongLongObjectOutLong( + envPtr, PlatformCallbackOp.ComputeTaskMap, memPtr, 0, 0, ses); } finally { leave(); @@ -304,13 +306,14 @@ public long computeJobCreate(long memPtr) { * * @param jobPtr Job pointer. * @param cancel Cancel flag. + * @param ses Platform compute task session proxy. */ - public void computeJobExecuteLocal(long jobPtr, long cancel) { + public void computeJobExecuteLocal(long jobPtr, long cancel, PlatformTargetProxy ses) { enter(); try { PlatformCallbackUtils.inLongLongLongObjectOutLong(envPtr, - PlatformCallbackOp.ComputeJobExecuteLocal, jobPtr, cancel, 0, null); + PlatformCallbackOp.ComputeJobExecuteLocal, jobPtr, cancel, 0, ses); } finally { leave(); @@ -321,12 +324,14 @@ public void computeJobExecuteLocal(long jobPtr, long cancel) { * Execute native job on a node other than where it was created. * * @param memPtr Memory pointer. + * @param ses Platform compute task session proxy. */ - public void computeJobExecute(long memPtr) { + public void computeJobExecute(long memPtr, PlatformTargetProxy ses) { enter(); try { - PlatformCallbackUtils.inLongOutLong(envPtr, PlatformCallbackOp.ComputeJobExecute, memPtr); + PlatformCallbackUtils.inLongLongLongObjectOutLong( + envPtr, PlatformCallbackOp.ComputeJobExecute, memPtr, 0, 0, ses); } finally { leave(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformAbstractJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformAbstractJob.java index fbde0d8aa9b882..eb6c36838fcce9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformAbstractJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformAbstractJob.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.binary.BinaryRawWriterEx; import org.apache.ignite.internal.processors.platform.PlatformContext; import org.apache.ignite.internal.processors.platform.PlatformProcessor; +import org.apache.ignite.internal.processors.platform.PlatformTargetProxy; import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStream; import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; @@ -130,13 +131,14 @@ protected boolean createJob(PlatformContext ctx) throws IgniteCheckedException { * * @param ctx Context. * @param cancel Cancel flag. + * @param ses Platform compute task session proxy. * @return Result. */ - protected Object runLocal(PlatformContext ctx, boolean cancel) { + protected Object runLocal(PlatformContext ctx, boolean cancel, PlatformTargetProxy ses) { // Local job, must execute it with respect to possible concurrent task completion. if (task.onJobLock()) { try { - ctx.gateway().computeJobExecuteLocal(ptr, cancel ? 1 : 0); + ctx.gateway().computeJobExecuteLocal(ptr, cancel ? 1 : 0, ses); return LOC_JOB_RES; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformClosureJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformClosureJob.java index 3bcb54995bf2f7..3c8db258fe0076 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformClosureJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformClosureJob.java @@ -73,7 +73,7 @@ public PlatformClosureJob(PlatformAbstractTask task, long ptr, Object job, Strin out.synchronize(); - ctx.gateway().computeJobExecute(mem.pointer()); + ctx.gateway().computeJobExecute(mem.pointer(), null); PlatformInputStream in = mem.input(); @@ -91,7 +91,7 @@ public PlatformClosureJob(PlatformAbstractTask task, long ptr, Object job, Strin // Local job execution. assert ptr != 0; - return runLocal(ctx, false); + return runLocal(ctx, false, null); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java index 90fe4ff28c40e1..5fef0eacb6b2ec 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java @@ -163,8 +163,10 @@ private PlatformCompute( long taskPtr = reader.readLong(); long topVer = reader.readLong(); String taskName = reader.readString(); + boolean taskSesFullSupport = reader.readBoolean(); - final PlatformFullTask task = new PlatformFullTask(platformCtx, platformGrp, taskPtr, topVer, taskName); + final PlatformFullTask task = new PlatformFullTask( + platformCtx, platformGrp, taskPtr, topVer, taskName, taskSesFullSupport); return executeNative0(task); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformComputeTaskSession.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformComputeTaskSession.java new file mode 100644 index 00000000000000..aee9888a3bfd22 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformComputeTaskSession.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.platform.compute; + +import java.util.Map; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.compute.ComputeTaskSession; +import org.apache.ignite.internal.binary.BinaryRawReaderEx; +import org.apache.ignite.internal.binary.BinaryRawWriterEx; +import org.apache.ignite.internal.processors.platform.PlatformAbstractTarget; +import org.apache.ignite.internal.processors.platform.PlatformContext; +import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; + +import static org.apache.ignite.internal.processors.platform.utils.PlatformUtils.readMap; + +/** {@link ComputeTaskSession} platform wrapper. */ +public class PlatformComputeTaskSession extends PlatformAbstractTarget { + /** "get attribute" operation code. */ + private static final int OP_GET_ATTRIBUTE = 1; + + /** "set attributes" operation code. */ + private static final int OP_SET_ATTRIBUTES = 2; + + /** Underlying compute task session. */ + private final ComputeTaskSession ses; + + /** + * Constructor. + * + * @param platformCtx Context. + * @param ses Underlying compute task session + */ + public PlatformComputeTaskSession(final PlatformContext platformCtx, final ComputeTaskSession ses) { + super(platformCtx); + + this.ses = ses; + } + + /** {@inheritDoc} */ + @Override public long processInStreamOutLong( + final int type, final BinaryRawReaderEx reader, final PlatformMemory mem) throws IgniteCheckedException { + + if (type == OP_SET_ATTRIBUTES) { + final Map attrs = readMap(reader); + + ses.setAttributes(attrs); + + return TRUE; + } + + return super.processInStreamOutLong(type, reader, mem); + } + + /** {@inheritDoc} */ + @Override public void processInStreamOutStream( + final int type, final BinaryRawReaderEx reader, final BinaryRawWriterEx writer) throws IgniteCheckedException { + + if (type == OP_GET_ATTRIBUTE) { + final Object key = reader.readObjectDetached(); + + final Object val = ses.getAttribute(key); + + writer.writeObjectDetached(val); + + return; + } + + super.processInStreamOutStream(type, reader, writer); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullJob.java index 4bf3b2f9d709d5..83a96dd0f8ba5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullJob.java @@ -22,13 +22,18 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.compute.ComputeTaskSession; import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.processors.platform.PlatformContext; import org.apache.ignite.internal.processors.platform.PlatformProcessor; +import org.apache.ignite.internal.processors.platform.PlatformTarget; +import org.apache.ignite.internal.processors.platform.PlatformTargetProxy; +import org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl; import org.apache.ignite.internal.processors.platform.memory.PlatformInputStream; import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStream; import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; +import org.apache.ignite.resources.TaskSessionResource; import org.jetbrains.annotations.Nullable; /** @@ -64,6 +69,10 @@ public class PlatformFullJob extends PlatformAbstractJob { /** Serialized job. */ private transient byte state; + /** Task session of this job. */ + @TaskSessionResource + private transient ComputeTaskSession ses; + /** * {@link Externalizable} support. */ @@ -111,8 +120,11 @@ public PlatformFullJob(PlatformContext ctx, PlatformAbstractTask task, long ptr, } try { + final PlatformTarget platformSes = new PlatformComputeTaskSession(ctx, ses); + final PlatformTargetProxy platformSesProxy = new PlatformTargetProxyImpl(platformSes, ctx); + if (task != null) - return runLocal(ctx, cancel); + return runLocal(ctx, cancel, platformSesProxy); else { try (PlatformMemory mem = ctx.memory().allocate()) { PlatformOutputStream out = mem.output(); @@ -122,7 +134,7 @@ public PlatformFullJob(PlatformContext ctx, PlatformAbstractTask task, long ptr, out.synchronize(); - ctx.gateway().computeJobExecute(mem.pointer()); + ctx.gateway().computeJobExecute(mem.pointer(), platformSesProxy); PlatformInputStream in = mem.input(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullTask.java index fd799b6b46fe1f..40beeea13283c6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformFullTask.java @@ -26,15 +26,20 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.ComputeTaskNoResultCache; +import org.apache.ignite.compute.ComputeTaskSession; import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.binary.BinaryRawWriterEx; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.processors.platform.PlatformContext; +import org.apache.ignite.internal.processors.platform.PlatformTarget; +import org.apache.ignite.internal.processors.platform.PlatformTargetProxy; +import org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl; import org.apache.ignite.internal.processors.platform.memory.PlatformInputStream; import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; import org.apache.ignite.internal.processors.platform.memory.PlatformMemoryManager; import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStream; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.resources.TaskSessionResource; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -55,6 +60,13 @@ public final class PlatformFullTask extends PlatformAbstractTask { /** Platform task name. */ private final String taskName; + /** {@code true} if distribution of the session attributes should be enabled. */ + private final boolean taskSesFullSupport; + + /** The task session. */ + @TaskSessionResource + private ComputeTaskSession ses; + /** * Constructor. * @@ -63,13 +75,21 @@ public final class PlatformFullTask extends PlatformAbstractTask { * @param taskPtr Pointer to the task in the native platform. * @param topVer Initial topology version. * @param taskName Task name. + * @param taskSesFullSupport {@code true} if distribution of the session attributes should be enabled. */ - public PlatformFullTask(PlatformContext ctx, ClusterGroup grp, long taskPtr, long topVer, String taskName) { + public PlatformFullTask( + PlatformContext ctx, + ClusterGroup grp, + long taskPtr, + long topVer, + String taskName, + boolean taskSesFullSupport) { super(ctx, taskPtr); this.grp = grp; this.topVer = topVer; this.taskName = taskName; + this.taskSesFullSupport = taskSesFullSupport; } /** {@inheritDoc} */ @@ -86,6 +106,9 @@ public PlatformFullTask(PlatformContext ctx, ClusterGroup grp, long taskPtr, lon PlatformMemoryManager memMgr = ctx.memory(); + final PlatformTarget platformSes = new PlatformComputeTaskSession(ctx, ses); + final PlatformTargetProxy platformSesProxy = new PlatformTargetProxyImpl(platformSes, ctx); + try (PlatformMemory mem = memMgr.allocate()) { PlatformOutputStream out = mem.output(); @@ -97,7 +120,7 @@ public PlatformFullTask(PlatformContext ctx, ClusterGroup grp, long taskPtr, lon out.synchronize(); - ctx.gateway().computeTaskMap(mem.pointer()); + ctx.gateway().computeTaskMap(mem.pointer(), platformSesProxy); PlatformInputStream in = mem.input(); @@ -113,6 +136,11 @@ public PlatformFullTask(PlatformContext ctx, ClusterGroup grp, long taskPtr, lon } } + /** {@code true} if distribution of session attributes should be enabled. */ + public boolean taskSessionFullSupport() { + return taskSesFullSupport; + } + /** * Write topology information. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java index f40d33ce6635e8..704067ca228dcd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java @@ -524,7 +524,7 @@ public PoolProcessor(GridKernalContext ctx) { DFLT_THREAD_KEEP_ALIVE_TIME, new LinkedBlockingQueue<>(), GridIoPolicy.UNDEFINED, - excHnd); + oomeHnd); snpExecSvc.allowCoreThreadTimeOut(true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java index 27c73d6a3f98e5..e4efdd3b38b7b3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java @@ -71,7 +71,6 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -1590,11 +1589,6 @@ public static boolean wasCancelled(Throwable e) { code = ((IgniteSQLException)e).statusCode(); } - else if (e instanceof TransactionDuplicateKeyException) { - code = IgniteQueryErrorCode.DUPLICATE_KEY; - - sqlState = IgniteQueryErrorCode.codeToSqlState(code); - } else { sqlState = SqlStateCode.INTERNAL_ERROR; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index 1761b7a1c20e8c..099acc974ea0bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -71,6 +71,7 @@ import org.apache.ignite.internal.processors.job.ComputeJobStatusEnum; import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; +import org.apache.ignite.internal.processors.platform.compute.PlatformFullTask; import org.apache.ignite.internal.processors.task.monitor.ComputeGridMonitor; import org.apache.ignite.internal.processors.task.monitor.ComputeTaskStatus; import org.apache.ignite.internal.processors.task.monitor.ComputeTaskStatusSnapshot; @@ -642,8 +643,9 @@ else if (task != null) { if (log.isDebugEnabled()) log.debug("Task deployment: " + dep); - boolean fullSup = dep != null && taskCls != null && - dep.annotation(taskCls, ComputeTaskSessionFullSupport.class) != null; + boolean fullSup = (dep != null && taskCls != null && + dep.annotation(taskCls, ComputeTaskSessionFullSupport.class) != null) || + (task instanceof PlatformFullTask && ((PlatformFullTask)task).taskSessionFullSupport()); Collection top = null; diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 7a45d5841c8ceb..70e1e4eab8c15f 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -2417,7 +2417,6 @@ org.apache.ignite.stream.StreamVisitor org.apache.ignite.stream.StreamVisitor$1 org.apache.ignite.transactions.TransactionConcurrency org.apache.ignite.transactions.TransactionDeadlockException -org.apache.ignite.transactions.TransactionDuplicateKeyException org.apache.ignite.transactions.TransactionException org.apache.ignite.transactions.TransactionHeuristicException org.apache.ignite.transactions.TransactionIsolation diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index b95452fc6f57a7..25021b2fe83d80 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -120,6 +120,14 @@ public class IgniteClusterSnapshotSelfTest extends AbstractSnapshotSelfTest { /** {@code true} if node should be started in separate jvm. */ protected volatile boolean jvm; + /** Any node failed. */ + private boolean failed; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setFailureHandler((ignite, ctx) -> failed = true); + } + /** @throws Exception If fails. */ @Before @Override public void beforeTestSnapshot() throws Exception { @@ -564,6 +572,28 @@ public void testClusterSnapshotExOnInitiatorLeft() throws Exception { } } + /** @throws Exception If fails. */ + @Test + public void testExceptionOnStartStage() throws Exception { + IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE); + + IgniteFuture fut = snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary); + + File snpDir = snp(ignite).snapshotLocalDir(SNAPSHOT_NAME); + + assertTrue(snpDir.mkdirs()); + + File snpMeta = new File(snpDir, IgniteSnapshotManager.snapshotMetaFileName(ignite.localNode().consistentId().toString())); + + assertTrue(snpMeta.createNewFile()); + + assertThrowsAnyCause(log, fut::get, IgniteException.class, "Snapshot metafile must not exist"); + + assertFalse(failed); + + createAndCheckSnapshot(ignite, SNAPSHOT_NAME); + } + /** @throws Exception If fails. */ @Test public void testSnapshotExistsException() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java index 38bf7dfb2c0d10..b518de85c09423 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java @@ -56,7 +56,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.StoredCacheData; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata; -import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -228,7 +227,9 @@ protected IgniteEx startGridAndFillCaches() throws Exception { } /** */ - protected T2> runDumpAsyncAndStopBeforeStart() throws IgniteInterruptedCheckedException { + protected T2> runDumpAsyncAndStopBeforeStart( + IgniteEx srv + ) throws IgniteInterruptedCheckedException { CountDownLatch latch = new CountDownLatch(1); List ignites = Ignition.allGrids(); @@ -244,7 +245,7 @@ protected T2> runDumpAsyncAndStopBeforeS }); } - IgniteInternalFuture dumpFut = runAsync(() -> createDump((IgniteEx)F.first(ignites))); + IgniteInternalFuture dumpFut = runAsync(() -> createDump(srv)); // Waiting while dump will be setup: task planned after change listener set. assertTrue(waitForCondition(() -> { @@ -344,82 +345,7 @@ void checkDump( assertEquals(nodes, nodesDirs.size()); - TestDumpConsumer cnsmr = new TestDumpConsumer() { - final Set keys = new HashSet<>(); - - final Set grpParts = new HashSet<>(); - - int dfltDumpSz; - - int grpDumpSz; - - @Override public void onCacheConfigs(Iterator caches) { - super.onCacheConfigs(caches); - - Set cachesFound = new HashSet<>(); - - caches.forEachRemaining(data -> { - String cacheName = data.config().getName(); - - assertTrue(cachesFound.add(cacheName)); - - assertEquals(cacheName, data.configuration().getName()); - - assertFalse(data.sql()); - - assertTrue(data.queryEntities().isEmpty()); - - if (cacheName.startsWith("cache-")) - assertEquals(GRP, data.configuration().getGroupName()); - else if (!cacheName.equals(DEFAULT_CACHE_NAME)) - throw new IgniteException("Unknown cache"); - }); - - assertEquals(expectedFoundCaches, cachesFound); - } - - @Override public void onPartition(int grp, int part, Iterator iter) { - if (onlyPrimary) - assertTrue(grpParts.add(toLong(grp, part))); - - if (grp == CU.cacheId(DEFAULT_CACHE_NAME)) { - while (iter.hasNext()) { - DumpEntry e = iter.next(); - - checkDefaultCacheEntry(e); - - keys.add((Integer)e.key()); - - dfltDumpSz++; - } - } - else { - while (iter.hasNext()) { - DumpEntry e = iter.next(); - - assertNotNull(e); - assertNotNull(e.version()); - assertNull(e.version().otherClusterVersion()); - - if (e.cacheId() == CU.cacheId(CACHE_0)) - assertEquals(USER_FACTORY.apply((Integer)e.key()), e.value()); - else - assertEquals(((Key)e.key()).getId() + "", ((Value)e.value()).getVal()); - - grpDumpSz++; - } - } - } - - @Override public void check() { - super.check(); - - assertEquals(expectedDfltDumpSz, dfltDumpSz); - assertEquals(expectedGrpDumpSz, grpDumpSz); - - IntStream.range(0, expectedCnt).forEach(key -> assertTrue(keys.contains(key))); - } - }; + TestDumpConsumer cnsmr = dumpConsumer(expectedFoundCaches, expectedDfltDumpSz, expectedGrpDumpSz, expectedCnt); new DumpReader( new DumpReaderConfiguration( @@ -440,14 +366,13 @@ else if (!cacheName.equals(DEFAULT_CACHE_NAME)) } /** */ - protected void checkDefaultCacheEntry(DumpEntry e) { - assertNotNull(e); - - Integer key = (Integer)e.key(); - - assertEquals(key, e.value()); - assertNotNull(e.version()); - assertNull(e.version().otherClusterVersion()); + protected TestDumpConsumer dumpConsumer( + Set expectedFoundCaches, + int expectedDfltDumpSz, + int expectedGrpDumpSz, + int expectedCnt + ) { + return new TestDumpConsumerImpl(expectedFoundCaches, expectedDfltDumpSz, expectedGrpDumpSz, expectedCnt); } /** */ @@ -577,6 +502,122 @@ public static KeystoreEncryptionSpi encryptionSpi() { return encSpi; } + /** */ + public class TestDumpConsumerImpl extends TestDumpConsumer { + /** */ + private final Set expectedFoundCaches; + + /** */ + private final int expectedDfltDumpSz; + + /** */ + private final int expectedGrpDumpSz; + + /** */ + private final int expectedCnt; + + /** */ + final Set keys = new HashSet<>(); + + /** */ + final Set grpParts = new HashSet<>(); + + /** */ + int dfltDumpSz; + + /** */ + int grpDumpSz; + + /** */ + protected TestDumpConsumerImpl(Set expectedFoundCaches, int expectedDfltDumpSz, int expectedGrpDumpSz, int expectedCnt) { + this.expectedFoundCaches = expectedFoundCaches; + this.expectedDfltDumpSz = expectedDfltDumpSz; + this.expectedGrpDumpSz = expectedGrpDumpSz; + this.expectedCnt = expectedCnt; + } + + /** {@inheritDoc} */ + @Override public void onCacheConfigs(Iterator caches) { + super.onCacheConfigs(caches); + + Set cachesFound = new HashSet<>(); + + caches.forEachRemaining(data -> { + String cacheName = data.config().getName(); + + assertTrue(cachesFound.add(cacheName)); + + assertEquals(cacheName, data.configuration().getName()); + + assertFalse(data.sql()); + + assertTrue(data.queryEntities().isEmpty()); + + if (cacheName.startsWith("cache-")) + assertEquals(GRP, data.configuration().getGroupName()); + else if (!cacheName.equals(DEFAULT_CACHE_NAME)) + throw new IgniteException("Unknown cache"); + }); + + assertEquals(expectedFoundCaches, cachesFound); + } + + /** {@inheritDoc} */ + @Override public void onPartition(int grp, int part, Iterator iter) { + if (onlyPrimary) + assertTrue(grpParts.add(toLong(grp, part))); + + if (grp == CU.cacheId(DEFAULT_CACHE_NAME)) { + while (iter.hasNext()) { + DumpEntry e = iter.next(); + + checkDefaultCacheEntry(e); + + keys.add((Integer)e.key()); + + dfltDumpSz++; + } + } + else { + while (iter.hasNext()) { + DumpEntry e = iter.next(); + + assertNotNull(e); + assertNotNull(e.version()); + assertNull(e.version().otherClusterVersion()); + + if (e.cacheId() == CU.cacheId(CACHE_0)) + assertEquals(USER_FACTORY.apply((Integer)e.key()), e.value()); + else + assertEquals(((Key)e.key()).getId() + "", ((Value)e.value()).getVal()); + + grpDumpSz++; + } + } + } + + /** {@inheritDoc} */ + @Override public void check() { + super.check(); + + assertEquals(expectedDfltDumpSz, dfltDumpSz); + assertEquals(expectedGrpDumpSz, grpDumpSz); + + IntStream.range(0, expectedCnt).forEach(key -> assertTrue(keys.contains(key))); + } + + /** */ + protected void checkDefaultCacheEntry(DumpEntry e) { + assertNotNull(e); + + Integer key = (Integer)e.key(); + + assertEquals(key, e.value()); + assertNotNull(e.version()); + assertNull(e.version().otherClusterVersion()); + } + } + /** */ public abstract static class TestDumpConsumer implements DumpConsumer { /** */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelfTest.java index 0a44519c7768b5..665b993d92e69f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelfTest.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -618,7 +619,7 @@ private boolean findValToFail(ByteBuffer srcBuf) { } }); - T2> latchAndFut = runDumpAsyncAndStopBeforeStart(); + T2> latchAndFut = runDumpAsyncAndStopBeforeStart(ign); cache.put(keyToFail, "test string"); @@ -647,7 +648,7 @@ private void checkDumpCleared(IgniteEx ign) throws IgniteCheckedException { private void doTestDumpWithExpiry() throws Exception { IgniteEx ign = startGridAndFillCaches(); - T2> latchAndFut = runDumpAsyncAndStopBeforeStart(); + T2> latchAndFut = runDumpAsyncAndStopBeforeStart(ign); Thread.sleep(TTL); @@ -677,7 +678,7 @@ private void doTestDumpWithExpiry() throws Exception { private void doTestConcurrentOperations(Consumer op) throws Exception { IgniteEx ign = startGridAndFillCaches(); - T2> latchAndFut = runDumpAsyncAndStopBeforeStart(); + T2> latchAndFut = runDumpAsyncAndStopBeforeStart(ign); // This operations will be catched by change listeners. Old value must be stored in dump. op.accept(ign); @@ -707,14 +708,24 @@ private void doTestConcurrentOperations(Consumer op) throws Exception } /** {@inheritDoc} */ - @Override protected void checkDefaultCacheEntry(DumpEntry e) { - super.checkDefaultCacheEntry(e); - - if (explicitTtl != null) { - assertTrue("Expire time must be set", e.expireTime() != 0); - assertTrue("Expire time must be in past", System.currentTimeMillis() >= e.expireTime()); - assertTrue("Expire time must be set during test run", System.currentTimeMillis() - getTestTimeout() < e.expireTime()); - } + @Override protected TestDumpConsumer dumpConsumer( + Set expectedFoundCaches, + int expectedDfltDumpSz, + int expectedGrpDumpSz, + int expectedCnt + ) { + return new TestDumpConsumerImpl(expectedFoundCaches, expectedDfltDumpSz, expectedGrpDumpSz, expectedCnt) { + /** {@inheritDoc} */ + @Override protected void checkDefaultCacheEntry(DumpEntry e) { + super.checkDefaultCacheEntry(e); + + if (explicitTtl != null) { + assertTrue("Expire time must be set", e.expireTime() != 0); + assertTrue("Expire time must be in past", System.currentTimeMillis() >= e.expireTime()); + assertTrue("Expire time must be set during test run", System.currentTimeMillis() - getTestTimeout() < e.expireTime()); + } + } + }; } /** */ diff --git a/modules/ducktests/src/main/java/java/net/DnsBlocker.java b/modules/ducktests/src/main/java/java/net/DnsBlocker.java deleted file mode 100644 index 8f939281220563..00000000000000 --- a/modules/ducktests/src/main/java/java/net/DnsBlocker.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package java.net; - -import java.io.File; -import java.io.FileNotFoundException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Scanner; - -/** */ -public class DnsBlocker { - /** */ - private static final String BLOCK_DNS_FILE = "/tmp/block_dns"; - - /** */ - public static final DnsBlocker INSTANCE = new DnsBlocker(); - - /** */ - private DnsBlocker() { - // No-op. - } - - /** - * Check and block hostname resolve request if needed. - * @param impl Implementation. - * @param hostname Hostname. - */ - public void onHostResolve(InetAddressImpl impl, String hostname) throws UnknownHostException { - if (!impl.loopbackAddress().getHostAddress().equals(hostname)) - check(hostname); - } - - /** - * Check and block address resolve request if needed. - * @param impl Implementation. - * @param addr Address. - */ - public void onAddrResolve(InetAddressImpl impl, byte[] addr) throws UnknownHostException { - if (!Arrays.equals(impl.loopbackAddress().getAddress(), addr)) - check(InetAddress.getByAddress(addr).toString()); - } - - /** */ - private void check(String req) throws UnknownHostException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - File file = new File(BLOCK_DNS_FILE); - - if (file.exists()) { - try { - Scanner scanner = new Scanner(file); - if (!scanner.hasNextLong()) - throw new RuntimeException("Wrong " + BLOCK_DNS_FILE + " file format"); - - long timeout = scanner.nextLong(); - - if (!scanner.hasNextBoolean()) - throw new RuntimeException("Wrong " + BLOCK_DNS_FILE + " file format"); - - boolean fail = scanner.nextBoolean(); - - // Can't use logger here, because class need to be in bootstrap classloader. - System.out.println(sdf.format(new Date()) + " [" + Thread.currentThread().getName() + - "] DNS request " + req + " blocked for " + timeout + " ms"); - - Thread.dumpStack(); - - Thread.sleep(timeout); - - if (fail) - throw new UnknownHostException(); - } - catch (InterruptedException | FileNotFoundException e) { - throw new RuntimeException(e); - } - } - else { - System.out.println(sdf.format(new Date()) + " [" + Thread.currentThread().getName() + - "] Passed DNS request " + req); - - Thread.dumpStack(); - } - } -} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/BlockingNameService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/BlockingNameService.java new file mode 100644 index 00000000000000..2c37056805d773 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/BlockingNameService.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.ducktest.tests.dns_failure_test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Scanner; + +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.startup.cmdline.CommandLineStartup; + +/** */ +public class BlockingNameService implements NameServiceHandler { + /** */ + private static final String BLOCK_DNS_FILE = "/tmp/block_dns"; + + /** Private {@code NameService} class to proxy. */ + private static Class nameSrvcCls; + + /** */ + private final InetAddress loopback; + + /** Original {@code NameService}. */ + private final Object origNameSrvc; + + /** + * @param origNameSrvc Original {@code NameService}. + */ + private BlockingNameService(Object origNameSrvc) { + loopback = InetAddress.getLoopbackAddress(); + this.origNameSrvc = origNameSrvc; + } + + /** Installs {@code BlockingNameService} as main {@code NameService} to JVM11. */ + private static void installJdk11() throws Exception { + Field nameSrvcFld = InetAddress.class.getDeclaredField("nameService"); + nameSrvcFld.setAccessible(true); + + nameSrvcCls = Class.forName("java.net.InetAddress$NameService"); + + BlockingNameService blkNameSrvc = new BlockingNameService(nameSrvcFld.get(InetAddress.class)); + + nameSrvcFld.set( + InetAddress.class, + Proxy.newProxyInstance(nameSrvcCls.getClassLoader(), new Class[] { nameSrvcCls }, blkNameSrvc)); + + System.out.println("Installed DnsBlocker as main NameService to JVM"); + } + + /** Installs {@code BlockingNameService} as main {@code NameService} to JVM8. */ + private static void installJdk8() throws Exception { + Field nameSrvcFld = InetAddress.class.getDeclaredField("nameServices"); + nameSrvcFld.setAccessible(true); + + nameSrvcCls = Class.forName("sun.net.spi.nameservice.NameService"); + + BlockingNameService blkNameSrvc = new BlockingNameService(((List)nameSrvcFld.get(InetAddress.class)).get(0)); + + nameSrvcFld.set(InetAddress.class, F.asList( + Proxy.newProxyInstance(InetAddress.class.getClassLoader(), new Class[] { nameSrvcCls }, blkNameSrvc) + )); + } + + /** */ + public static void main(String[] args) throws Exception { + String jdkVer = U.jdkVersion(); + + if ("1.8".equals(jdkVer)) + installJdk8(); + else if ("11".equals(jdkVer)) + installJdk11(); + else + throw new IllegalArgumentException("Unsupported JDK version: " + jdkVer); + + System.out.println("Installed BlockingNameService as main NameService to JVM"); + + CommandLineStartup.main(args); + } + + /** */ + @Override public InetAddress[] lookupAllHostAddr(String hostname) throws UnknownHostException { + if (!loopback.getHostAddress().equals(hostname)) + check(hostname); + + try { + Method lookupAllHostAddr = nameSrvcCls.getDeclaredMethod("lookupAllHostAddr", String.class); + lookupAllHostAddr.setAccessible(true); + + return (InetAddress[])lookupAllHostAddr.invoke(origNameSrvc, hostname); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** */ + @Override public String getHostByAddr(byte[] addr) throws UnknownHostException { + if (!Arrays.equals(loopback.getAddress(), addr)) + check(InetAddress.getByAddress(addr).toString()); + + try { + Method getHostByAddr = nameSrvcCls.getDeclaredMethod("getHostByAddr", byte[].class); + getHostByAddr.setAccessible(true); + + return (String)getHostByAddr.invoke(origNameSrvc, addr); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** */ + private void check(String req) throws UnknownHostException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + File file = new File(BLOCK_DNS_FILE); + + if (file.exists()) { + try { + Scanner scanner = new Scanner(file); + if (!scanner.hasNextLong()) + throw new RuntimeException("Wrong " + BLOCK_DNS_FILE + " file format"); + + long timeout = scanner.nextLong(); + + if (!scanner.hasNextBoolean()) + throw new RuntimeException("Wrong " + BLOCK_DNS_FILE + " file format"); + + boolean fail = scanner.nextBoolean(); + + // Can't use logger here, because class need to be in bootstrap classloader. + System.out.println(sdf.format(new Date()) + " [" + Thread.currentThread().getName() + + "] DNS request " + req + " blocked for " + timeout + " ms"); + + Thread.dumpStack(); + + Thread.sleep(timeout); + + if (fail) + throw new UnknownHostException(); + } + catch (InterruptedException | FileNotFoundException e) { + throw new RuntimeException(e); + } + } + else { + System.out.println(sdf.format(new Date()) + " [" + Thread.currentThread().getName() + + "] Passed DNS request " + req); + + Thread.dumpStack(); + } + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/NameServiceHandler.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/NameServiceHandler.java new file mode 100644 index 00000000000000..3b37605e3b82d1 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/dns_failure_test/NameServiceHandler.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.dns_failure_test; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** Handler for {@code java.net.InetAddress$NameService}. */ +interface NameServiceHandler extends InvocationHandler { + /** Intercepts {@code NameService#lookupAllHostAddr}. */ + public InetAddress[] lookupAllHostAddr(String host) throws UnknownHostException; + + /** Intercepts {@code NameService#getHostByAddr}. */ + public String getHostByAddr(byte[] addr) throws UnknownHostException; + + /** Delegate {@code NameService} methods to {@link BlockingNameService}. */ + @Override public default Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + String name = method.getName(); + + if ("lookupAllHostAddr".equals(name)) + return lookupAllHostAddr((String)args[0]); + else if ("getHostByAddr".equals(name)) + return getHostByAddr((byte[])args[0]); + else + throw new UnsupportedOperationException("Unsupported method: " + name); + } +} diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index da93c8771881cb..50a34ad97aea17 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -27,6 +27,6 @@ class IgniteService(IgniteAwareService): APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup" def __init__(self, context, config, num_nodes, jvm_opts=None, merge_with_default=True, startup_timeout_sec=60, - shutdown_timeout_sec=60, modules=None): - super().__init__(context, config, num_nodes, startup_timeout_sec, shutdown_timeout_sec, self.APP_SERVICE_CLASS, + shutdown_timeout_sec=60, modules=None, main_java_class=APP_SERVICE_CLASS): + super().__init__(context, config, num_nodes, startup_timeout_sec, shutdown_timeout_sec, main_java_class, modules, jvm_opts=jvm_opts, merge_with_default=merge_with_default) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index 81ff3fe58a95a4..e8c2c036b8b36c 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -245,7 +245,8 @@ def envs(self): return { 'EXCLUDE_TEST_CLASSES': 'true', 'IGNITE_LOG_DIR': self.service.log_dir, - 'USER_LIBS': ":".join(self.libs()) + 'USER_LIBS': ":".join(self.libs()), + "MAIN_CLASS": self.service.main_java_class } def config_file_path(self): @@ -356,6 +357,3 @@ def command(self, node): def config_file_path(self): return self.service.config_file if self.service.config.service_type == IgniteServiceType.NODE \ else self.service.thin_client_config_file - - def envs(self): - return {**super().envs(), **{"MAIN_CLASS": self.service.main_java_class}} diff --git a/modules/ducktests/tests/ignitetest/tests/dns_failure_test.py b/modules/ducktests/tests/ignitetest/tests/dns_failure_test.py index 2a9e676a8ad170..d2ccf6c39a1c40 100644 --- a/modules/ducktests/tests/ignitetest/tests/dns_failure_test.py +++ b/modules/ducktests/tests/ignitetest/tests/dns_failure_test.py @@ -16,7 +16,6 @@ """ Module contains DNS service failure test. """ -import os import socket from ducktape.mark import defaults @@ -27,7 +26,6 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster -from ignitetest.services.utils.jvm_utils import java_major_version from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import IgniteVersion, DEV_BRANCH @@ -110,19 +108,8 @@ def __prepare_service(self, ignite_config, num_nodes=1): self.test_context, ignite_config, startup_timeout_sec=120, - num_nodes=num_nodes) - - bootclasspath = list(map(lambda lib: os.path.join(lib, "classes"), ignite.spec._module_libs("ducktests"))) - - # Note: Support of impl.prefix property was removed since java 18. - ignite.spec.jvm_opts.append("-Dimpl.prefix=BlockingDns") - - java_version = ignite.java_version() - - if java_major_version(java_version) > 8: - ignite.spec.jvm_opts.append("\"--patch-module java.base=" + ":".join(bootclasspath) + "\"") - else: - ignite.spec.jvm_opts.append("-Xbootclasspath/a:" + ":".join(bootclasspath)) + num_nodes=num_nodes, + main_java_class="org.apache.ignite.internal.ducktest.tests.dns_failure_test.BlockingNameService") return ignite diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index f355dd70919228..091a15c9230f6e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -46,6 +46,8 @@ import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.EventType; import org.apache.ignite.events.SqlQueryExecutionEvent; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInternalFuture; @@ -692,6 +694,10 @@ private ResultSet executeSqlQuery(final H2PooledConnection conn, final PreparedS if (e.getErrorCode() == ErrorCode.STATEMENT_WAS_CANCELED) throw new QueryCancelledException(); + if (e.getErrorCode() == ErrorCode.OUT_OF_MEMORY) { + ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } + if (e.getCause() instanceof IgniteSQLException) throw (IgniteSQLException)e.getCause(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java index e5d3d4e90b2740..1cfd895fac456d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java @@ -51,7 +51,6 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.h2.util.DateTimeUtils; import org.h2.util.LocalDateTimeUtils; import org.h2.value.Value; @@ -196,6 +195,8 @@ public static UpdateResult processSelectResult(UpdatePlan plan, Iterable private static long dmlDoInsert(UpdatePlan plan, Iterable> cursor, int pageSize) throws IgniteCheckedException { GridCacheContext cctx = plan.cacheContext(); + final String errMsg = "Failed to INSERT some keys because they are already in cache [keys="; + // If we have just one item to put, just do so if (plan.rowCount() == 1) { IgniteBiTuple t = plan.processRow(cursor.iterator().next()); @@ -208,7 +209,7 @@ private static long dmlDoInsert(UpdatePlan plan, Iterable> cursor, int p if (cctx.cache().putIfAbsent(t.getKey(), t.getValue())) return 1; else - throw new TransactionDuplicateKeyException("Duplicate key during INSERT [key=" + t.getKey() + ']'); + throw new IgniteSQLException(errMsg + '[' + t.getKey() + "]]", DUPLICATE_KEY); } } else { @@ -226,8 +227,7 @@ private static long dmlDoInsert(UpdatePlan plan, Iterable> cursor, int p SQLException resEx = snd.error(); if (!F.isEmpty(snd.failedKeys())) { - String msg = "Failed to INSERT some keys because they are already in cache " + - "[keys=" + snd.failedKeys() + ']'; + String msg = errMsg + snd.failedKeys() + ']'; SQLException dupEx = new SQLException(msg, SqlStateCode.CONSTRAINT_VIOLATION); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/metric/SqlStatisticsUserQueriesFastTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/metric/SqlStatisticsUserQueriesFastTest.java index 36101662046a69..e5f957cb5b0c8a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/metric/SqlStatisticsUserQueriesFastTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/metric/SqlStatisticsUserQueriesFastTest.java @@ -30,7 +30,6 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.running.RunningQueryManager; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.junit.Test; import static org.apache.ignite.internal.util.IgniteUtils.resolveIgnitePath; @@ -147,8 +146,8 @@ public void testDmlSuccess() { assertMetricsIncrementedOnlyOnReducer(() -> GridTestUtils.assertThrowsAnyCause( log, () -> cache.query(new SqlFieldsQuery("INSERT INTO TAB VALUES(5, 'I will NOT be inserted')")).getAll(), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"), + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"), "failed"); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java index c0dba13e96823e..cfe59d025ebf20 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java @@ -20,7 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.UUID; -import java.util.concurrent.Callable; +import java.util.function.Consumer; import javax.cache.CacheException; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.query.SqlFieldsQuery; @@ -172,25 +172,41 @@ public void testPrimitives() { } /** - * + * Checks whether it's impossible to insert duplicate in single key statement. */ @Test - public void testDuplicateKeysException() { + public void testDuplicateSingleKey() { + doTestDuplicate( + p -> p.query(new SqlFieldsQuery("insert into Integer(_key, _val) values (1, ?), " + + "(?, 5), (5, 6)").setArgs(2, 3)), + new SqlFieldsQuery("insert into Integer(_key, _val) values (?, ?)").setArgs(3, 5) + ); + } + + /** + * Checks whether it's impossible to insert duplicate in multiple keys statement. + */ + @Test + public void testDuplicateMultipleKeys() { + doTestDuplicate( + p -> p.put(3, 5), + new SqlFieldsQuery("insert into Integer(_key, _val) values (1, ?), " + + "(?, 4), (5, 6)").setArgs(2, 3) + ); + } + + /** + * + */ + private void doTestDuplicate(Consumer> initAction, SqlFieldsQuery sql) { final IgniteCache p = ignite(0).cache("I2I"); p.clear(); - p.put(3, 5); - - GridTestUtils.assertThrows(log, new Callable() { - /** {@inheritDoc} */ - @Override public Void call() throws Exception { - p.query(new SqlFieldsQuery("insert into Integer(_key, _val) values (1, ?), " + - "(?, 4), (5, 6)").setArgs(2, 3)); + initAction.accept(p); - return null; - } - }, CacheException.class, "Failed to INSERT some keys because they are already in cache [keys=[3]]"); + GridTestUtils.assertThrows(log, () -> p.query(sql), CacheException.class, + "Failed to INSERT some keys because they are already in cache [keys=[3]]"); assertEquals(2, (int)p.get(1)); assertEquals(5, (int)p.get(3)); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlInsertValidationSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlInsertValidationSelfTest.java index 94113a735592f3..c3cb6546f75dd5 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlInsertValidationSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlInsertValidationSelfTest.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.junit.Test; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -149,8 +148,8 @@ public void testIncorrectComplex() { GridTestUtils.assertThrows(log(), () -> execute("INSERT INTO FORGOTTEN_KEY_FLDS(FK1, FK2, FV1, FV2) VALUES (8,9,10,11)"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteInsertNullableDuplicatesSqlTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteInsertNullableDuplicatesSqlTest.java index ecfa5df71e70c6..caf1c2c67fd9ec 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteInsertNullableDuplicatesSqlTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteInsertNullableDuplicatesSqlTest.java @@ -28,7 +28,6 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.transactions.TransactionDuplicateKeyException; import org.junit.Test; import static org.apache.ignite.testframework.GridTestUtils.assertThrows; @@ -69,13 +68,13 @@ public void testInsertKeyWithNullKeyParts() { assertThrows(log, () -> sql("insert into test (id1, id2, val) values (1, null, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (id1, val) values (1, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertEquals(sql("SELECT * FROM test").getAll().size(), 1); } @@ -90,18 +89,18 @@ public void testInsertKeyWithNullKeys() { assertThrows(log, () -> sql("insert into test (id1, val) values (null, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (id2, val) values (null, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (id2, id1, val) values (null, null, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertEquals(sql("SELECT * FROM test").getAll().size(), 1); } @@ -115,8 +114,8 @@ public void testInsertKeyWhenKeyIsNotSet() { sql("insert into test (val) values (1);"); assertThrows(log, () -> sql("insert into test (val) values (1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); } /** @@ -131,13 +130,13 @@ public void testInsertKeyWithNullKeyPartsDefault() { assertThrows(log, () -> sql("insert into test (id1, val) values (0, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (val) values (2);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); List> sql = sql("select * from test order by val asc;").getAll(); @@ -183,13 +182,13 @@ public void testInsertKeyWithNullKeyPartsDefaultCacheApi() { assertThrows(log, () -> sql("insert into test (id1, val) values (0, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (val) values (2);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); List> sql = sql("select * from test order by val asc;").getAll(); @@ -232,13 +231,13 @@ public void testInsertKeyWithNullKeyParts2() { assertThrows(log, () -> sql("insert into test (id1, id2, val) values (1, null, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertThrows(log, () -> sql("insert into test (id1, val) values (1, 1);"), - TransactionDuplicateKeyException.class, - "Duplicate key during INSERT"); + IgniteSQLException.class, + "Failed to INSERT some keys because they are already in cache"); assertEquals(sql("SELECT * FROM test").getAll().size(), 1); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/IgniteQueryOOMTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/IgniteQueryOOMTestSuite.java index d57a607fe45429..16843deeafb84b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/IgniteQueryOOMTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/IgniteQueryOOMTestSuite.java @@ -28,6 +28,7 @@ //Query history. QueryOOMWithoutQueryParallelismTest.class, QueryOOMWithQueryParallelismTest.class, + OOMLeadsTest.class, }) public class IgniteQueryOOMTestSuite { } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/OOMLeadsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/OOMLeadsTest.java new file mode 100644 index 00000000000000..b684f09aebdae4 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/oom/OOMLeadsTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.oom; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import org.apache.ignite.IgniteException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.StopNodeFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy; +import org.junit.Test; + +import static org.apache.ignite.testframework.GridTestUtils.assertThrows; + +/** Out of memory handling. */ +public class OOMLeadsTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected List additionalRemoteJvmArgs() { + return Arrays.asList("-Xmx64m", "-Xms64m"); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return true; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + IgniteProcessProxy.killAll(); + + super.afterTest(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setFailureHandler(new StopNodeFailureHandler()); + + return cfg; + } + + /** Check correct handling for Out of memory. */ + @Test + public void testOOMQueryHandling() throws Exception { + startGrids(2); + + // stop local jvm node + stopGrid(0); + + // check non oom sql processed correctly + runQuery("select x, space(100+x) as av from system_range(1, 1) group by av"); + + // oom lead sql + assertThrows(null, () -> + runQuery("select x, space(10000000+x) as av from system_range(1, 1000) group by av"), + IgniteException.class, "Out of memory"); + + IgniteEx grd = grid(1); + + assertTrue(GridTestUtils.waitForCondition(() -> !((IgniteProcessProxy)grd).getProcess().getProcess().isAlive(), 10_000)); + } + + /** */ + private void runQuery(String sql) throws Exception { + try (Connection c = DriverManager.getConnection( + "jdbc:ignite:thin://127.0.0.1:10800..10850/")) { + try (Statement stmt = c.createStatement()) { + stmt.execute(sql); + } + } + } +} diff --git a/modules/json/README.txt b/modules/json/README.txt new file mode 100644 index 00000000000000..5eae369e8b30a1 --- /dev/null +++ b/modules/json/README.txt @@ -0,0 +1,4 @@ +Apache Ignite Json Module +------------------------ + +Apache Ignite json module provides classes and configurations for integration with external tools via json format. \ No newline at end of file diff --git a/modules/json/pom.xml b/modules/json/pom.xml new file mode 100644 index 00000000000000..4cf92c4b8421f8 --- /dev/null +++ b/modules/json/pom.xml @@ -0,0 +1,100 @@ + + + + + + + 4.0.0 + + + org.apache.ignite + ignite-parent-internal + ${revision} + ../../parent-internal/pom.xml + + + ignite-json + + http://ignite.apache.org + + + + ${project.groupId} + ignite-core + + + + ${project.groupId} + ignite-spring + test + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.apache.logging.log4j + log4j-core + test + + + + ${project.groupId} + ignite-core + test-jar + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + false + + + + + + diff --git a/modules/json/src/main/java/org/apache/ignite/dump/JsonDumpConsumer.java b/modules/json/src/main/java/org/apache/ignite/dump/JsonDumpConsumer.java new file mode 100644 index 00000000000000..0bf4fe28906011 --- /dev/null +++ b/modules/json/src/main/java/org/apache/ignite/dump/JsonDumpConsumer.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.dump; + +import java.io.IOException; +import java.util.Iterator; +import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryType; +import org.apache.ignite.cache.CacheEntryVersion; +import org.apache.ignite.cdc.TypeMapping; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; +import org.apache.ignite.internal.processors.cache.StoredCacheData; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpConsumerKernalContextAware; + +/** + * Dump consumer that outputs entries in json format. + */ +public class JsonDumpConsumer implements DumpConsumerKernalContextAware { + /** Ignite specific object mapper. */ + private IgniteObjectMapper mapper; + + /** {@inheritDoc} */ + @Override public void start(GridKernalContext ctx) { + mapper = new IgniteObjectMapper(ctx); + } + + /** {@inheritDoc} */ + @Override public void onMappings(Iterator mappings) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void onTypes(Iterator types) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void onCacheConfigs(Iterator caches) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void onPartition(int grp, int part, Iterator data) { + data.forEachRemaining(entry -> { + try { + System.out.println(mapper.writeValueAsString(new PrintableDumpEntry(entry))); + } + catch (IOException e) { + throw new IgniteException(e); + } + }); + } + + /** {@inheritDoc} */ + @Override public void stop() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void start() { + // No-op. + } + + /** */ + private static class PrintableDumpEntry { + /** */ + private final DumpEntry e; + + /** */ + public PrintableDumpEntry(DumpEntry e) { + this.e = e; + } + + /** @see DumpEntry#cacheId() */ + public int getCacheId() { + return e.cacheId(); + } + + /** @see DumpEntry#expireTime() */ + public long getExpireTime() { + return e.expireTime(); + } + + /** @see DumpEntry#version() */ + public PrintableCacheEntryVersion getVersion() { + return new PrintableCacheEntryVersion(e.version()); + } + + /** @see DumpEntry#key() */ + public Object getKey() { + return e.key(); + } + + /** @see DumpEntry#value() */ + public Object getValue() { + return e.value(); + } + } + + /** */ + private static class PrintableCacheEntryVersion { + /** */ + private final CacheEntryVersion v; + + /** */ + public PrintableCacheEntryVersion(CacheEntryVersion v) { + this.v = v; + } + + /** @see CacheEntryVersion#order() */ + public long getOrder() { + return v.order(); + } + + /** @see CacheEntryVersion#nodeOrder() */ + public int getNodeOrder() { + return v.nodeOrder(); + } + + /** @see CacheEntryVersion#clusterId() */ + public byte getClusterId() { + return v.clusterId(); + } + + /** @see CacheEntryVersion#topologyVersion() */ + public int getTopologyVersion() { + return v.topologyVersion(); + } + + /** @see CacheEntryVersion#otherClusterVersion() */ + public PrintableCacheEntryVersion otherClusterVersion() { + return new PrintableCacheEntryVersion(v.otherClusterVersion()); + } + } +} diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/IgniteBinaryObjectJsonDeserializer.java b/modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteBinaryObjectJsonDeserializer.java similarity index 98% rename from modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/IgniteBinaryObjectJsonDeserializer.java rename to modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteBinaryObjectJsonDeserializer.java index a6d0d6fbe0958b..08723facfa78ab 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/IgniteBinaryObjectJsonDeserializer.java +++ b/modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteBinaryObjectJsonDeserializer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.rest.protocols.http.jetty; +package org.apache.ignite.internal.jackson; import java.io.IOException; import java.util.Collections; diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java b/modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteObjectMapper.java similarity index 96% rename from modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java rename to modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteObjectMapper.java index fc3966a8548690..804f5159532104 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java +++ b/modules/json/src/main/java/org/apache/ignite/internal/jackson/IgniteObjectMapper.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.rest.protocols.http.jetty; +package org.apache.ignite.internal.jackson; import java.io.IOException; import java.sql.Date; @@ -59,21 +59,24 @@ /** * Custom object mapper for HTTP REST API. */ -public class GridJettyObjectMapper extends ObjectMapper { +public class IgniteObjectMapper extends ObjectMapper { + /** Date format. */ + public static final DateFormat DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT, Locale.US); + /** * Default constructor. */ - public GridJettyObjectMapper() { + public IgniteObjectMapper() { this(null); } /** * @param ctx Defines a kernal context to enable deserialization into the Ignite binary object. */ - GridJettyObjectMapper(GridKernalContext ctx) { + public IgniteObjectMapper(GridKernalContext ctx) { super(null, new CustomSerializerProvider(), null); - setDateFormat(DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT, Locale.US)); + setDateFormat(DATE_FORMAT); SimpleModule module = new SimpleModule(); @@ -98,7 +101,8 @@ public GridJettyObjectMapper() { IgnitePredicate clsFilter = ctx.marshallerContext().classNameFilter(); - setDefaultTyping(new RestrictedTypeResolverBuilder(clsFilter).init(JsonTypeInfo.Id.CLASS, null)); + if (clsFilter != null) + setDefaultTyping(new RestrictedTypeResolverBuilder(clsFilter).init(JsonTypeInfo.Id.CLASS, null)); } configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); diff --git a/modules/json/src/main/java/org/apache/ignite/internal/management/cache/scan/JsonCacheScanTaskFormat.java b/modules/json/src/main/java/org/apache/ignite/internal/management/cache/scan/JsonCacheScanTaskFormat.java new file mode 100644 index 00000000000000..34443987f74198 --- /dev/null +++ b/modules/json/src/main/java/org/apache/ignite/internal/management/cache/scan/JsonCacheScanTaskFormat.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.cache.scan; + +import java.util.Collections; +import java.util.List; +import javax.cache.Cache; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; + +/** + * This format prints cache objects in json format. + */ +public class JsonCacheScanTaskFormat implements CacheScanTaskFormat { + /** */ + public static final String NAME = "json"; + + /** */ + private final ObjectMapper mapper = new IgniteObjectMapper(); + + /** {@inheritDoc} */ + @Override public String name() { + return NAME; + } + + /** {@inheritDoc} */ + @Override public List titles(Cache.Entry first) { + return Collections.singletonList("data"); + } + + /** {@inheritDoc} */ + @Override public List row(Cache.Entry e) { + try { + return Collections.singletonList(mapper.writeValueAsString(e)); + } + catch (JsonProcessingException ex) { + throw new IgniteException(ex); + } + } +} diff --git a/modules/json/src/main/resources/META-INF/services/org.apache.ignite.internal.management.cache.scan.CacheScanTaskFormat b/modules/json/src/main/resources/META-INF/services/org.apache.ignite.internal.management.cache.scan.CacheScanTaskFormat new file mode 100644 index 00000000000000..cfbbc776498344 --- /dev/null +++ b/modules/json/src/main/resources/META-INF/services/org.apache.ignite.internal.management.cache.scan.CacheScanTaskFormat @@ -0,0 +1 @@ +org.apache.ignite.internal.management.cache.scan.JsonCacheScanTaskFormat \ No newline at end of file diff --git a/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocTokenType.java b/modules/json/src/test/java/org/apache/ignite/dump/IgniteJsonSuite.java similarity index 71% rename from modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocTokenType.java rename to modules/json/src/test/java/org/apache/ignite/dump/IgniteJsonSuite.java index f29f3c11c79f72..b82cbe8ef2bc99 100644 --- a/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocTokenType.java +++ b/modules/json/src/test/java/org/apache/ignite/dump/IgniteJsonSuite.java @@ -15,24 +15,17 @@ * limitations under the License. */ -package org.apache.ignite.tools.ant.beautifier; +package org.apache.ignite.dump; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; /** - * Lexical token type. + * Json test suite. */ -enum GridJavadocTokenType { - /** HTML instruction. */ - TOKEN_INSTR, - - /** HTML comment. */ - TOKEN_COMM, - - /** HTML open tag. */ - TOKEN_OPEN_TAG, - - /** HTML close tag. */ - TOKEN_CLOSE_TAG, - - /** HTML text. */ - TOKEN_TEXT +@RunWith(Suite.class) +@Suite.SuiteClasses({ + JsonDumpConsumerTest.class +}) +public class IgniteJsonSuite { } diff --git a/modules/json/src/test/java/org/apache/ignite/dump/JsonDumpConsumerTest.java b/modules/json/src/test/java/org/apache/ignite/dump/JsonDumpConsumerTest.java new file mode 100644 index 00000000000000..06aff788042b10 --- /dev/null +++ b/modules/json/src/test/java/org/apache/ignite/dump/JsonDumpConsumerTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.dump; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ignite.cache.CacheEntryVersion; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpConsumerKernalContextAware; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.IgniteCacheDumpSelfTest; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.platform.model.ACL; +import org.apache.ignite.platform.model.AccessLevel; +import org.apache.ignite.platform.model.Key; +import org.apache.ignite.platform.model.Role; +import org.apache.ignite.platform.model.User; +import org.apache.ignite.platform.model.Value; +import org.jetbrains.annotations.NotNull; + +/** */ +public class JsonDumpConsumerTest extends IgniteCacheDumpSelfTest { + /** {@inheritDoc} */ + @Override protected TestDumpConsumer dumpConsumer( + Set expectedFoundCaches, + int expectedDfltDumpSz, + int expectedGrpDumpSz, + int expectedCnt + ) { + return new TestJsonDumpConsumer(expectedFoundCaches, expectedDfltDumpSz, expectedGrpDumpSz, expectedCnt); + } + + /** */ + public class TestJsonDumpConsumer extends TestDumpConsumerImpl implements DumpConsumerKernalContextAware { + /** */ + private final JsonDumpConsumer jsonDumpConsumer = new JsonDumpConsumer(); + + /** */ + protected TestJsonDumpConsumer(Set expectedFoundCaches, int expectedDfltDumpSz, int expectedGrpDumpSz, int expectedCnt) { + super(expectedFoundCaches, expectedDfltDumpSz, expectedGrpDumpSz, expectedCnt); + } + + /** {@inheritDoc} */ + @Override public void start(GridKernalContext ctx) { + jsonDumpConsumer.start(ctx); + start(); + } + + /** {@inheritDoc} */ + @Override public void onPartition(int grp, int part, Iterator data) { + ByteArrayOutputStream testOut = new ByteArrayOutputStream((int)(16 * U.MB)); + + PrintStream out = System.out; + + System.setOut(new PrintStream(testOut)); + + // Parse entries from System.out. + Scanner sc; + + try { + // Print out all entries to System.out. + jsonDumpConsumer.onPartition(grp, part, data); + + sc = new Scanner(testOut.toString()); + } + finally { + System.setOut(out); + } + + TypeReference> typeRef = new TypeReference>() { + // No-op. + }; + + ObjectMapper mapper = new ObjectMapper(); + + super.onPartition(grp, part, new Iterator() { + @Override public boolean hasNext() { + return sc.hasNextLine(); + } + + @Override public DumpEntry next() { + try { + Map entryFromJson = mapper.readValue(sc.nextLine(), typeRef); + + return new DumpEntry() { + @Override public int cacheId() { + return Integer.parseInt(entryFromJson.get("cacheId").toString()); + } + + @Override public long expireTime() { + return Long.parseLong(entryFromJson.get("expireTime").toString()); + } + + @Override public CacheEntryVersion version() { + return JsonDumpConsumerTest.version((Map)entryFromJson.get("version")); + } + + @Override public Object key() { + if (cacheId() == CU.cacheId(DEFAULT_CACHE_NAME) || cacheId() == CU.cacheId(CACHE_0)) + return Integer.parseInt(entryFromJson.get("key").toString()); + + Map key = (Map)entryFromJson.get("key"); + + return new Key(Long.parseLong(key.get("id").toString())); + } + + @Override public Object value() { + if (cacheId() == CU.cacheId(DEFAULT_CACHE_NAME)) + return Integer.valueOf(entryFromJson.get("value").toString()); + else if (cacheId() == CU.cacheId(CACHE_0)) { + Map val = (Map)entryFromJson.get("value"); + Map role = (Map)val.get("role"); + + return new User( + (Integer)val.get("id"), + ACL.valueOf(val.get("acl").toString()), + new Role( + role.get("name").toString(), + AccessLevel.valueOf(role.get("accessLevel").toString()) + ) + ); + } + else if (cacheId() == CU.cacheId(CACHE_1)) { + Map val = (Map)entryFromJson.get("value"); + + return new Value(val.get("val").toString()); + } + + throw new IllegalArgumentException("Unknown cache: " + cacheId()); + } + }; + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + }); + } + } + + /** */ + private static CacheEntryVersion version(Map version) { + return new CacheEntryVersion() { + @Override public long order() { + return Long.parseLong(version.get("order").toString()); + } + + @Override public int nodeOrder() { + return Integer.parseInt(version.get("nodeOrder").toString()); + } + + @Override public byte clusterId() { + return Byte.parseByte(version.get("clusterId").toString()); + } + + @Override public int topologyVersion() { + return Integer.parseInt(version.get("topologyVersion").toString()); + } + + @Override public CacheEntryVersion otherClusterVersion() { + return version.containsKey("otherClusterVersion") + ? version((Map)version.get("otherClusterVersion")) + : null; + } + + @Override public int compareTo(@NotNull CacheEntryVersion o) { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/modules/platforms/cpp/core/src/impl/ignite_environment.cpp b/modules/platforms/cpp/core/src/impl/ignite_environment.cpp index 4c84f16273499c..d845c458169fe3 100644 --- a/modules/platforms/cpp/core/src/impl/ignite_environment.cpp +++ b/modules/platforms/cpp/core/src/impl/ignite_environment.cpp @@ -198,15 +198,6 @@ namespace ignite break; } - case OperationCallback::COMPUTE_JOB_EXECUTE: - { - SharedPointer mem = env->Get()->GetMemory(val); - - env->Get()->ComputeJobExecute(mem); - - break; - } - case OperationCallback::COMPUTE_JOB_DESTROY: { env->Get()->ComputeJobDestroy(val); @@ -319,6 +310,15 @@ namespace ignite switch (type) { + case OperationCallback::COMPUTE_JOB_EXECUTE: + { + SharedPointer mem = env->Get()->GetMemory(val1); + + env->Get()->ComputeJobExecute(mem); + + break; + } + case OperationCallback::COMPUTE_JOB_EXECUTE_LOCAL: { env->Get()->ComputeJobExecuteLocal(val1); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeTaskSessionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeTaskSessionTest.cs new file mode 100644 index 00000000000000..e336b52a6f1d8a --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeTaskSessionTest.cs @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Compute +{ + using System.Collections.Generic; + using System.Linq; + using Apache.Ignite.Core.Compute; + using Apache.Ignite.Core.Resource; + using NUnit.Framework; + using static TestUtils; + + /// + /// Tests for + /// + public class ComputeTaskSessionTest + { + /// + /// Data stored in a session by a task is available in the compute job created by the task on another node. + /// + [Test] + public void DistributesTaskSessionAttributeRemotely() + { + // Given an Ignite cluster consisting of server and client nodes + using var ignored = Ignition.Start(GetIgniteConfiguration("server1")); + using var ignite = Ignition.Start(GetIgniteConfiguration("client1", true)); + + // When the user executes a task setting a session attribute and creating a job getting the attribute + const string attrName = "attr1"; + const int attrValue = 123; + var task = new SessionAttributeSetterTask(attrName); + var sessionValue = ignite.GetCompute().Execute(task, attrValue); + + // Then the task returns the same attribute value + Assert.AreEqual(attrValue, sessionValue); + } + + /// + /// Data stored in session by a task is available in the compute job created by the task on the same node. + /// + [Test] + public void DistributesTaskSessionAttributeLocally() + { + // Given a single node Ignite cluster + using var ignite = Ignition.Start(GetIgniteConfiguration("server1")); + + // When the user executes a task setting a session attribute and creating a job getting the attribute + const string attrName = "attr1"; + const int attrValue = 123; + var task = new SessionAttributeSetterTask(attrName); + var sessionValue = ignite.GetCompute().Execute(task, attrValue); + + // Then the task returns the same attribute value + Assert.AreEqual(attrValue, sessionValue); + } + + private static IgniteConfiguration GetIgniteConfiguration(string igniteName, bool isClient = false) => + new IgniteConfiguration + { + ClientMode = isClient, + ConsistentId = igniteName, + IgniteInstanceName = igniteName, + DiscoverySpi = GetStaticDiscovery(), + JvmOptions = TestJavaOptions() + }; + + /// + /// Sets the specified session attribute and creates one . + /// + [ComputeTaskSessionFullSupport] + private class SessionAttributeSetterTask : ComputeTaskSplitAdapter + { + private readonly string _attrName; +#pragma warning disable 649 + [TaskSessionResource] private IComputeTaskSession _taskSession; +#pragma warning restore 649 + + public SessionAttributeSetterTask(string attrName) + { + _attrName = attrName; + } + + /// + public override int Reduce(IList> results) => results.Select(res => res.Data).Sum(); + + /// + protected override ICollection> Split(int gridSize, int attrValue) + { + _taskSession.SetAttributes(new KeyValuePair(_attrName, attrValue)); + return new List> {new SessionAttributeGetterJob(_attrName)}; + } + } + + /// + /// Returns the specified session attribute. + /// + private class SessionAttributeGetterJob : ComputeJobAdapter + { + private readonly string _attrName; +#pragma warning disable 649 + [TaskSessionResource] private IComputeTaskSession _taskSession; +#pragma warning restore 649 + + public SessionAttributeGetterJob(string attrName) + { + _attrName = attrName; + } + + /// + public override int Execute() => _taskSession.GetAttribute(_attrName); + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTaskSession.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTaskSession.cs new file mode 100644 index 00000000000000..e94e6b3e95ab9e --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTaskSession.cs @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Compute +{ + using System.Collections.Generic; + + /// + /// Stores custom compute task attributes. Specific compute task implementations must be annotated with the + /// to enable distributing the task attributes to the compute + /// jobs that the task creates. + /// + public interface IComputeTaskSession + { + /// + /// Gets the value of the given key or null if the key does not exist. + /// + TV GetAttribute(TK key); + + /// + /// Stores the collection of attributes. + /// + void SetAttributes(params KeyValuePair[] attrs); + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs index af2f793000a9ec..8e9a6a1dd13ec1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs @@ -237,6 +237,7 @@ public Future Execute(IComputeTask + /// Injects compute task session into wrapped object. + /// + [TaskSessionResource] + public void InjectTaskSession(IComputeTaskSession taskSes) + { + // Propagate injection + ResourceProcessor.InjectComputeTaskSession(Job, taskSes); + } + /// /// Gets the inner job. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeJobHolder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeJobHolder.cs index 7d83380a8a3ad2..228b66809fe662 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeJobHolder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeJobHolder.cs @@ -15,6 +15,8 @@ * limitations under the License. */ +using Apache.Ignite.Core.Compute; + namespace Apache.Ignite.Core.Impl.Compute { using System; @@ -71,11 +73,12 @@ public ComputeJobHolder(IIgniteInternal grid, IComputeJob job) /// Executes local job. /// /// Cancel flag. + /// Compute task session [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "User code can throw any exception type.")] - public void ExecuteLocal(bool cancel) + public void ExecuteLocal(bool cancel, IComputeTaskSession taskSes) { - ComputeRunner.InjectResources(_ignite, _job); + ComputeRunner.InjectResources(_ignite, taskSes, _job); var nodeId = _ignite.GetIgnite().GetCluster().GetLocalNode().Id; @@ -99,9 +102,10 @@ public void ExecuteLocal(bool cancel) /// /// Whether the job must be cancelled. /// Stream. - public void ExecuteRemote(PlatformMemoryStream stream, bool cancel) + /// Compute task session + public void ExecuteRemote(PlatformMemoryStream stream, bool cancel, IComputeTaskSession taskSes) { - ComputeRunner.ExecuteJobAndWriteResults(_ignite, stream, _job, _ => Execute0(cancel)); + ComputeRunner.ExecuteJobAndWriteResults(_ignite, taskSes, stream, _job, _ => Execute0(cancel)); } /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeRunner.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeRunner.cs index 10170fbcfc68c1..0f57b2f4d8dc12 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeRunner.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeRunner.cs @@ -15,6 +15,8 @@ * limitations under the License. */ +using Apache.Ignite.Core.Compute; + namespace Apache.Ignite.Core.Impl.Compute { using System; @@ -37,7 +39,11 @@ internal static class ComputeRunner /// [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "User code can throw any exception type.")] - public static void ExecuteJobAndWriteResults(IIgniteInternal ignite, PlatformMemoryStream stream, T job, + public static void ExecuteJobAndWriteResults( + IIgniteInternal ignite, + IComputeTaskSession taskSes, + PlatformMemoryStream stream, + T job, Func execFunc) { Debug.Assert(stream != null); @@ -46,7 +52,7 @@ public static void ExecuteJobAndWriteResults(IIgniteInternal ignite, Platform Debug.Assert(execFunc != null); // 0. Inject resources. - InjectResources(ignite, job); + InjectResources(ignite, taskSes, job); // 1. Execute job. object res; @@ -84,14 +90,18 @@ public static void ExecuteJobAndWriteResults(IIgniteInternal ignite, Platform /// /// Performs compute-specific resource injection. /// - public static void InjectResources(IIgniteInternal ignite, object job) + public static void InjectResources(IIgniteInternal ignite, IComputeTaskSession computeTaskSession, object job) { var injector = job as IComputeResourceInjector; + // Injecting task session is supported only for if (injector != null) injector.Inject(ignite); else + { ResourceProcessor.Inject(job, ignite); + ResourceProcessor.InjectComputeTaskSession(job, computeTaskSession); + } } } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskHolder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskHolder.cs index e2bed73e195710..32d301d4ecde59 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskHolder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskHolder.cs @@ -43,8 +43,9 @@ internal interface IComputeTaskHolder /// Perform map step. /// /// Stream with IN data (topology info) and for OUT data (map result). + /// Optional compute task session /// Map with produced jobs. - void Map(PlatformMemoryStream stream); + void Map(PlatformMemoryStream stream, IComputeTaskSession taskSes); /// /// Process local job result. @@ -104,6 +105,9 @@ internal class ComputeTaskHolder : IComputeTaskHolder /** Task future. */ private readonly Future _fut = new Future(); + /** Resource descriptor. */ + private readonly ResourceTypeDescriptor _resDesc; + /** Jobs whose results are cached. */ private ISet _resJobs; @@ -125,23 +129,26 @@ public ComputeTaskHolder(Ignite grid, ComputeImpl compute, IComputeTask + public bool TaskSessionFullSupport { get; } /** */ [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "User code can throw any exception")] - public void Map(PlatformMemoryStream stream) + public void Map(PlatformMemoryStream stream, IComputeTaskSession taskSes) { IList subgrid; @@ -149,6 +156,9 @@ public void Map(PlatformMemoryStream stream) var ignite = (Ignite) prj.Ignite; + // 0. Inject session + _resDesc.InjectTaskSession(_task, taskSes); + // 1. Unmarshal topology info if topology changed. var reader = prj.Marshaller.StartUnmarshal(stream); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSession.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSession.cs new file mode 100644 index 00000000000000..4acbee11095794 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSession.cs @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Compute +{ + using System.Collections.Generic; + using Apache.Ignite.Core.Compute; + using Binary; + + /// + /// Implements by delegating the implementation to the Java side. + /// + internal class ComputeTaskSession : PlatformTargetAdapter, IComputeTaskSession + { + /// + /// Operation codes + /// + private enum Op + { + GetAttribute = 1, + SetAttributes = 2 + } + + /// + /// Initializes a new instance of the class. + /// + public ComputeTaskSession(IPlatformTargetInternal target) : base(target) + { + } + + /// + public TV GetAttribute(TK key) => + DoOutInOp((int) Op.GetAttribute, w => w.Write(key)); + + /// + public void SetAttributes(params KeyValuePair[] attrs) => + DoOutOp((int) Op.SetAttributes, writer => writer.WriteDictionary(attrs)); + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSessionFullSupportAttribute.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSessionFullSupportAttribute.cs new file mode 100644 index 00000000000000..01541f760052bc --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeTaskSessionFullSupportAttribute.cs @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Compute +{ + using System; + using static System.AttributeTargets; + + /// + /// Enables distributing 's attributes from + /// to that the task creates. + /// implementations must be annotated with the + /// to enable the features depending on the + /// attributes. + /// By default attributes and checkpoints are disabled for performance reasons. + /// + [AttributeUsage(Class | Struct)] + public sealed class ComputeTaskSessionFullSupportAttribute : Attribute + { + // No-op. + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceProcessor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceProcessor.cs index 6e37006968ba8f..9846372ecce156 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceProcessor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceProcessor.cs @@ -15,6 +15,8 @@ * limitations under the License. */ +using Apache.Ignite.Core.Compute; + namespace Apache.Ignite.Core.Impl.Resource { using System; @@ -91,5 +93,20 @@ public static void InjectStoreSession(ICacheStore store, ICacheStoreSession ses) Descriptor(store.GetType()).InjectStoreSession(store, ses); } + + /// + /// Injects compute task session into a job or task. + /// + /// Compute job or task + /// Compute task session + public static void InjectComputeTaskSession(object target, IComputeTaskSession taskSes) + { + if (target != null) + { + var desc = Descriptor(target.GetType()); + + desc.InjectTaskSession(target, taskSes); + } + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceTypeDescriptor.cs index 609ccfd565ac1f..ab43723cdde7f4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Resource/ResourceTypeDescriptor.cs @@ -36,17 +36,26 @@ internal class ResourceTypeDescriptor /** Attribute type: StoreSessionResourceAttribute. */ private static readonly Type TypAttrStoreSes = typeof(StoreSessionResourceAttribute); + /** Attribute type: TaskSessionResourceAttribute. */ + private static readonly Type TypAttrTaskSes = typeof(TaskSessionResourceAttribute); + /** Type: IGrid. */ private static readonly Type TypIgnite = typeof(IIgnite); /** Type: ICacheStoreSession. */ private static readonly Type TypStoreSes = typeof (ICacheStoreSession); + /** Type: IComputeTaskSession. */ + private static readonly Type TypTaskSes = typeof (IComputeTaskSession); + /** Type: ComputeTaskNoResultCacheAttribute. */ private static readonly Type TypComputeTaskNoResCache = typeof(ComputeTaskNoResultCacheAttribute); + /** Type: ComputeTaskSessionFullSupportAttribute. */ + private static readonly Type TypComputeTaskSessionFullSupport = typeof(ComputeTaskSessionFullSupportAttribute); + /** Cached binding flags. */ - private const BindingFlags Flags = BindingFlags.Instance | BindingFlags.Public | + private const BindingFlags Flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; /** Ignite injectors. */ @@ -54,10 +63,13 @@ internal class ResourceTypeDescriptor /** Session injectors. */ private readonly IList _storeSesInjectors; - + + /** Compute task session injectors. */ + private readonly IList _taskSesInjectors; + /** Task "no result cache" flag. */ private readonly bool _taskNoResCache; - + /// /// Constructor. /// @@ -66,20 +78,23 @@ internal ResourceTypeDescriptor(Type type) { Collector gridCollector = new Collector(TypAttrIgnite, TypIgnite); Collector storeSesCollector = new Collector(TypAttrStoreSes, TypStoreSes); + var taskSesCollector = new Collector(TypAttrTaskSes, TypTaskSes); Type curType = type; while (curType != null) { - CreateInjectors(curType, gridCollector, storeSesCollector); + CreateInjectors(curType, gridCollector, storeSesCollector, taskSesCollector); curType = curType.BaseType; } _igniteInjectors = gridCollector.Injectors; _storeSesInjectors = storeSesCollector.Injectors; + _taskSesInjectors = taskSesCollector.Injectors; _taskNoResCache = ContainsAttribute(type, TypComputeTaskNoResCache, true); + TaskSessionFullSupport = ContainsAttribute(type, TypComputeTaskSessionFullSupport, true); } /// @@ -102,6 +117,19 @@ public void InjectStoreSession(object target, ICacheStoreSession ses) Inject0(target, ses, _storeSesInjectors); } + /// + /// Inject compute task session. + /// + /// Target. + /// Compute task session. + public void InjectTaskSession(object target, IComputeTaskSession ses) + { + if (ses != null) + { + Inject0(target, ses, _taskSesInjectors); + } + } + /// /// Perform injection. /// @@ -127,7 +155,10 @@ public bool TaskNoResultCache return _taskNoResCache; } } - + + /// + public bool TaskSessionFullSupport { get; } + /// /// Create gridInjectors for the given type. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs index b2fcc96b009df9..6c5dde106611da 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs @@ -493,11 +493,12 @@ private long OnAffinityTopologyVersionChanged( #region IMPLEMENTATION: COMPUTE - private long ComputeTaskMap(long memPtr) + private long ComputeTaskMap(long memPtr, long ignored, long ignored2, void* sesPtr) { using (PlatformMemoryStream stream = IgniteManager.Memory.Get(memPtr).GetStream()) { - Task(stream.ReadLong()).Map(stream); + var ses = TaskSession(sesPtr); + Task(stream.ReadLong()).Map(stream, ses); return 0; } @@ -562,15 +563,18 @@ private long ComputeJobCreate(long memPtr) } } - private long ComputeJobExecuteLocal(long jobPtr, long cancel, long unused, void* arg) + private long ComputeJobExecuteLocal(long jobPtr, long cancel, long unused, void* sesPtr) { - Job(jobPtr).ExecuteLocal(cancel == 1); + var ses = TaskSession(sesPtr); + + Job(jobPtr).ExecuteLocal(cancel == 1, ses); return 0; } - private long ComputeJobExecute(long memPtr) + private long ComputeJobExecute(long memPtr, long ignored, long ignored2, void* sesPtr) { + var ses = TaskSession(sesPtr); using (PlatformMemoryStream stream = IgniteManager.Memory.Get(memPtr).GetStream()) { var job = Job(stream.ReadLong()); @@ -579,7 +583,7 @@ private long ComputeJobExecute(long memPtr) stream.Reset(); - job.ExecuteRemote(stream, cancel); + job.ExecuteRemote(stream, cancel, ses); } return 0; @@ -615,6 +619,18 @@ private IComputeTaskHolder Task(long taskPtr) return _handleRegistry.Get(taskPtr); } + private IComputeTaskSession TaskSession(void* sesPtr) + { + if (sesPtr == null) + { + return null; + } + + var sesRef = _jvm.AttachCurrentThread().NewGlobalRef((IntPtr) sesPtr); + var sesTarget = new PlatformJniTarget(sesRef, _ignite.Marshaller); + return new ComputeTaskSession(sesTarget); + } + /// /// Get compute job using it's GC handle pointer. /// @@ -640,7 +656,7 @@ private long ComputeOutFuncExecute(long memPtr) stream.Reset(); var invoker = DelegateTypeDescriptor.GetComputeOutFunc(func.GetType()); - ComputeRunner.ExecuteJobAndWriteResults(_ignite, stream, func, invoker); + ComputeRunner.ExecuteJobAndWriteResults(_ignite, null, stream, func, invoker); } return 0; @@ -660,7 +676,7 @@ private long ComputeActionExecute(long memPtr) stream.Reset(); - ComputeRunner.ExecuteJobAndWriteResults(_ignite, stream, action, act => + ComputeRunner.ExecuteJobAndWriteResults(_ignite, null, stream, action, act => { act.Invoke(); return null; diff --git a/modules/ducktests/src/main/java/java/net/BlockingDnsInet4AddressImpl.java b/modules/platforms/dotnet/Apache.Ignite.Core/Resource/TaskSessionResourceAttribute.cs similarity index 60% rename from modules/ducktests/src/main/java/java/net/BlockingDnsInet4AddressImpl.java rename to modules/platforms/dotnet/Apache.Ignite.Core/Resource/TaskSessionResourceAttribute.cs index 9e98aa37c8a15a..905bb271deb5e3 100644 --- a/modules/ducktests/src/main/java/java/net/BlockingDnsInet4AddressImpl.java +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Resource/TaskSessionResourceAttribute.cs @@ -14,21 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package java.net; -/** */ -public class BlockingDnsInet4AddressImpl extends Inet4AddressImpl { - /** {@inheritDoc} */ - @Override public InetAddress[] lookupAllHostAddr(String hostname) throws UnknownHostException { - DnsBlocker.INSTANCE.onHostResolve(this, hostname); +namespace Apache.Ignite.Core.Resource +{ + using System; + using Compute; + using static System.AttributeTargets; - return super.lookupAllHostAddr(hostname); - } - - /** {@inheritDoc} */ - @Override public String getHostByAddr(byte[] addr) throws UnknownHostException { - DnsBlocker.INSTANCE.onAddrResolve(this, addr); - - return super.getHostByAddr(addr); + /// + /// Injects into implementations of + /// and . + /// + [AttributeUsage(Field | Method | Property)] + public sealed class TaskSessionResourceAttribute : Attribute + { + // No-op. } } diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml index f326f428621be5..2eb640b145babc 100644 --- a/modules/rest-http/pom.xml +++ b/modules/rest-http/pom.xml @@ -47,6 +47,11 @@ ignite-core + + ${project.groupId} + ignite-json + + commons-lang commons-lang @@ -95,24 +100,6 @@ 3.1.0 - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - org.slf4j slf4j-api diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java index ba09e7ff04b01a..e9550d7a35888c 100644 --- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java +++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java @@ -51,6 +51,8 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.jackson.IgniteBinaryObjectJsonDeserializer; +import org.apache.ignite.internal.jackson.IgniteObjectMapper; import org.apache.ignite.internal.processors.cache.CacheConfigurationOverride; import org.apache.ignite.internal.processors.rest.GridRestCommand; import org.apache.ignite.internal.processors.rest.GridRestProtocolHandler; @@ -172,7 +174,7 @@ public class GridJettyRestHandler extends AbstractHandler { this.hnd = hnd; this.authChecker = authChecker; this.log = ctx.log(getClass()); - this.jsonMapper = new GridJettyObjectMapper(ctx); + this.jsonMapper = new IgniteObjectMapper(ctx); // Init default page and favicon. try { diff --git a/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocAntTask.java b/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocAntTask.java index e142c658a77035..df4b0ad9d44d1c 100644 --- a/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocAntTask.java +++ b/modules/tools/src/main/java/org/apache/ignite/tools/ant/beautifier/GridJavadocAntTask.java @@ -27,9 +27,8 @@ import java.io.OutputStream; import java.io.Reader; import java.io.StringWriter; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collection; import jodd.jerry.Jerry; import jodd.lagarto.dom.LagartoDOMBuilder; import org.apache.tools.ant.BuildException; @@ -44,9 +43,6 @@ public class GridJavadocAntTask extends MatchingTask { /** Directory. */ private File dir; - /** CSS file name. */ - private String css; - /** Whether to verify JavaDoc HTML. */ private boolean verify = true; @@ -61,17 +57,6 @@ public void setDir(File dir) { this.dir = dir; } - /** - * Sets CSS file name. - * - * @param css CSS file name to set. - */ - public void setCss(String css) { - assert css != null; - - this.css = css; - } - /** * Sets whether to verify JavaDoc HTML. * @@ -105,11 +90,7 @@ private void close(Closeable closeable) { if (dir == null) throw new BuildException("'dir' attribute must be specified."); - if (css == null) - throw new BuildException("'css' attribute must be specified."); - log("dir=" + dir, Project.MSG_DEBUG); - log("css=" + css, Project.MSG_DEBUG); DirectoryScanner scanner = getDirectoryScanner(dir); @@ -152,16 +133,18 @@ private String prepareErrorSummary(ArrayList errMsgs) { } /** - * Processes file (validating and cleaning up Javadoc's HTML). + * Processes file (validating Javadoc's HTML). * - * @param file File to cleanup. + * @param fileName File to validate. * @throws IOException Thrown in case of any I/O error. * @throws IllegalArgumentException In JavaDoc HTML validation failed. */ - private void processFile(String file) throws IOException { - assert file != null; + private void processFile(String fileName) throws IOException { + assert fileName != null; - String fileContent = readFileToString(file, Charset.forName("UTF-8")); + File file = new File(fileName); + + String fileContent = readFileToString(file); if (verify) { // Parse HTML. @@ -171,8 +154,16 @@ private void processFile(String file) throws IOException { .configure(cfg -> cfg.setErrorLogEnabled(false)) ).parse(fileContent); - // TODO https://issues.apache.org/jira/browse/IGNITE-13202 Check also index.html file. - if (file.endsWith("overview-summary.html")) { + String jdkVer = System.getProperty("java.specification.version"); + + boolean jdk11 = "11".equals(jdkVer); + + if (!jdk11 && !"1.8".equals(jdkVer)) { + throw new IllegalArgumentException("GridJavadocAntTask isn't tested for java versions after 11. " + + "Please check html rendering of documentation package groups works correctly and remove this exception then."); + } + + if (file.getName().equals(jdk11 ? "index.html" : "overview-summary.html")) { // Try to find Other Packages section. Jerry otherPackages = doc.find("div.contentContainer table.overviewSummary caption span:contains('Other Packages')"); @@ -185,186 +176,39 @@ private void processFile(String file) throws IOException { "Please add packages description to parent/pom.xml into (maven-javadoc-plugin) / " + " / "); } - } - else if (!isViewHtml(file)) { - // Try to find a class description block. - Jerry descBlock = doc.find("div.contentContainer .description"); - - if (descBlock.size() == 0) - throw new IllegalArgumentException("Class doesn't have description in file: " + file); - } - } - - GridJavadocCharArrayLexReader lexer = new GridJavadocCharArrayLexReader(fileContent.toCharArray()); - - Collection toks = new ArrayList<>(); - - StringBuilder tokBuf = new StringBuilder(); - - int ch; - - while ((ch = lexer.read()) != GridJavadocCharArrayLexReader.EOF) { - // Instruction, tag or comment. - if (ch == '<') { - if (tokBuf.length() > 0) { - toks.add(new GridJavadocToken(GridJavadocTokenType.TOKEN_TEXT, tokBuf.toString())); - - tokBuf.setLength(0); - } - - tokBuf.append('<'); - - ch = lexer.read(); - - if (ch == GridJavadocCharArrayLexReader.EOF) - throw new IOException("Unexpected EOF: " + file); - - // Instruction or comment. - if (ch == '!') { - for (; ch != GridJavadocCharArrayLexReader.EOF && ch != '>'; ch = lexer.read()) - tokBuf.append((char)ch); - - if (ch == GridJavadocCharArrayLexReader.EOF) - throw new IOException("Unexpected EOF: " + file); - - assert ch == '>'; - - tokBuf.append('>'); - - String val = tokBuf.toString(); - - toks.add(new GridJavadocToken(val.startsWith(" ${javadoc.opts} --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED - false @@ -1027,6 +1081,14 @@ + + + org.apache.maven.plugins + maven-javadoc-plugin + + ${javadoc.opts} --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED + + diff --git a/pom.xml b/pom.xml index de4d928f2dc82f..d220357d6ab13b 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ modules/web modules/urideploy modules/indexing + modules/json modules/rest-http modules/jta modules/log4j2 @@ -160,7 +161,51 @@ ${basedir}/target/javadoc core - org.apache.ignite -exclude org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.communication.tcp.internal:org.apache.ignite.spi.discovery.zk.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest + + + + + io + :io.* + :org.apache.calcite + :org.apache.calcite.* + :org.jsr166 + :org.jsr166.* + :org.mindrot + :org.mindrot.* + + + :org.apache.ignite.codegen + :org.apache.ignite.codegen.* + :org.apache.ignite.examples + :org.apache.ignite.examples.* + :org.apache.ignite.internal + :org.apache.ignite.internal.* + :org.apache.ignite.schema + :org.apache.ignite.schema.* + :org.apache.ignite.tests + :org.apache.ignite.tests.* + :org.apache.ignite.tools + :org.apache.ignite.tools.* + :org.apache.ignite.util + :org.apache.ignite.util.* + :org.apache.ignite.spi.discovery.tcp.messages + :org.apache.ignite.spi.discovery.tcp.messages.* + :org.apache.ignite.spi.discovery.tcp.internal + :org.apache.ignite.spi.discovery.tcp.internal.* + :org.apache.ignite.spi.communication.tcp.internal + :org.apache.ignite.spi.communication.tcp.internal.* + :org.apache.ignite.spi.discovery.zk.internal + :org.apache.ignite.spi.discovery.zk.internal.* + :org.apache.ignite.spi.deployment.uri.scanners + :org.apache.ignite.spi.deployment.uri.scanners.* + :org.apache.ignite.spi.deployment.uri.tasks + :org.apache.ignite.spi.deployment.uri.tasks.* + :org.apache.ignite.yardstick + :org.apache.ignite.yardstick.* + :org.apache.ignite.webtest + :org.apache.ignite.webtest.* + @@ -194,7 +239,7 @@ - +