Skip to content

Commit

Permalink
Merge remote-tracking branch 'refs/remotes/apache/master' into ignite…
Browse files Browse the repository at this point in the history
  • Loading branch information
NSAmelchev committed Jul 11, 2024
2 parents 8a25b69 + 2448ffa commit d8e8e09
Show file tree
Hide file tree
Showing 87 changed files with 2,997 additions and 1,155 deletions.
76 changes: 76 additions & 0 deletions docs/_docs/code-snippets/dotnet/JobScheduling.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using Apache.Ignite.Core;
using Apache.Ignite.Core.Compute;
using Apache.Ignite.Core.Resource;

namespace dotnet_helloworld
{
public class JobScheduling
{
public void Priority()
{
// tag::priority[]
// PriorityQueueCollisionSpi must be configured in the Spring XML configuration file ignite-helloworld.xml
var cfg = new IgniteConfiguration
{
SpringConfigUrl = "ignite-helloworld.xml"
};

// Start a node.
using var ignite = Ignition.Start(cfg);
// end::priority[]
}

// tag::task-priority[]
// Compute tasks must be annotated with the ComputeTaskSessionFullSupport attribute to support distributing
// the task's session attributes to compute jobs that the task creates.
[ComputeTaskSessionFullSupport]
public class MyUrgentTask : ComputeTaskSplitAdapter<int, bool, bool>
{
// Auto-injected task session.
[TaskSessionResource] private IComputeTaskSession _taskSes;

/// <inheritdoc />
protected override ICollection<IComputeJob<bool>> Split(int gridSize, int arg)
{
// Set high task priority.
_taskSes.SetAttribute("grid.task.priority", 10);

var jobs = new List<IComputeJob<bool>>(gridSize);

for (var i = 1; i <= gridSize; i++)
{
jobs.Add(new MyUrgentJob());
}

// These jobs will be executed with higher priority.
return jobs;
}

/// <inheritdoc />
public override bool Reduce(IList<IComputeJobResult<bool>> results) => results.All(r => r.Data);
}
// end::task-priority[]

private class MyUrgentJob : ComputeJobAdapter<bool>
{
public override bool Execute() => true;
}
}
}
12 changes: 10 additions & 2 deletions docs/_docs/distributed-computing/job-scheduling.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
= Job Scheduling

:javaFile: {javaCodeDir}/JobScheduling.java
:csharpFile: {csharpCodeDir}/JobScheduling.cs

When jobs arrive at the destination node, they are submitted to a thread pool and scheduled for execution in random order.
However, you can change job ordering by configuring `CollisionSpi`.
Expand Down Expand Up @@ -70,9 +71,16 @@ tab:C++[unsupported]

Task priorities are set in the link:distributed-computing/map-reduce#distributed-task-session[task session] via the `grid.task.priority` attribute. If no priority is assigned to a task, then the default priority of 0 is used.


[tabs]
--
tab:Java[]
[source, java]
----
include::{javaFile}[tag=task-priority,indent=0]
----

tab:C#/.NET[]
[source,csharp]
----
include::{csharpFile}[tag=task-priority,indent=0]
----
--
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,19 @@ You should consider the nature of your transactions, the rate of change of your

Custom conflict resolver can be set via `conflictResolver` and allows to compare or merge the conflict data in any required way.

=== Conflict Resolver Metrics

The Ignite's built-in `CacheVersionConflictResolverPluginProvider` provides the following metrics:

[cols="35%,65%",opts="header"]
|===
|Name |Description
| `AcceptedCount` | Count of accepted entries.
| `RejectedCount` | Count of rejected entries.
|===

These metrics are registered under `conflict-resolver` registry for each node configured with this plugin.

=== Configuration example
Configuration is done via Ignite node plugin:

Expand Down
5 changes: 5 additions & 0 deletions modules/bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@
<artifactId>ignite-opencensus</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ignite-json</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ignite-rest-http</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion modules/calcite/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
<immutables.version>2.8.2</immutables.version>
<janino.version>3.1.8</janino.version>
<javacc-maven-plugin>2.4</javacc-maven-plugin>
<jsonpath.version>2.7.0</jsonpath.version>
<jsonpath.version>2.9.0</jsonpath.version>
<reflections.version>0.10.2</reflections.version>
<commons.math.version>3.6.1</commons.math.version>
</properties>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.calcite.adapter.enumerable.RexImpTable;
import org.apache.calcite.linq4j.tree.ConstantExpression;
import org.apache.calcite.linq4j.tree.ConstantUntypedNull;
Expand Down Expand Up @@ -221,14 +220,14 @@ public static Expression convert(Expression operand, Type fromType, Type toType)
final Primitive fromBox = Primitive.ofBox(fromType);
final Primitive fromPrimitive = Primitive.of(fromType);
final boolean fromNumber = fromType instanceof Class
&& Number.class.isAssignableFrom((Class)fromType);
&& Number.class.isAssignableFrom((Class<?>)fromType);
if (fromType == String.class) {
if (toPrimitive != null) {
if (toPrimitive.isFixedNumeric())
return IgniteExpressions.parseStringChecked(operand, toPrimitive);

switch (toPrimitive) {
case CHAR:
case SHORT:
case INT:
case LONG:
case FLOAT:
case DOUBLE:
// Generate "SqlFunctions.toShort(x)".
Expand All @@ -245,6 +244,9 @@ public static Expression convert(Expression operand, Type fromType, Type toType)
}
}
if (toBox != null) {
if (toBox.isFixedNumeric())
operand = IgniteExpressions.parseStringChecked(operand, toBox);

switch (toBox) {
case CHAR:
// Generate "SqlFunctions.toCharBoxed(x)".
Expand Down Expand Up @@ -277,12 +279,11 @@ public static Expression convert(Expression operand, Type fromType, Type toType)

if (fromPrimitive != null) {
// E.g. from "float" to "double"
return Expressions.convert_(
operand, toPrimitive.primitiveClass);
return IgniteExpressions.convertChecked(operand, fromPrimitive, toPrimitive);
}
if (fromNumber || fromBox == Primitive.CHAR) {
if (fromNumber) {
// Generate "x.shortValue()".
return Expressions.unbox(operand, toPrimitive);
return IgniteExpressions.unboxChecked(operand, fromBox, toPrimitive);
}
else {
// E.g. from "Object" to "short".
Expand All @@ -300,7 +301,7 @@ else if (fromNumber && toBox != null) {
Expressions.equal(operand, RexImpTable.NULL_EXPR),
RexImpTable.NULL_EXPR,
Expressions.box(
Expressions.unbox(operand, toBox),
IgniteExpressions.unboxChecked(operand, fromBox, toBox),
toBox));
}
else if (fromPrimitive != null && toBox != null) {
Expand All @@ -322,7 +323,7 @@ else if (fromPrimitive != null && toBox != null) {
// Convert it first and generate "Byte.valueOf((byte)x)"
// Because there is no method "Byte.valueOf(int)" in Byte
return Expressions.box(
Expressions.convert_(operand, toBox.primitiveClass),
IgniteExpressions.convertChecked(operand, fromPrimitive, toBox),
toBox);
}
// Convert datetime types to internal storage type:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.internal.processors.query.calcite.exec.exp;

import java.lang.reflect.Type;
import java.math.BigDecimal;
import org.apache.calcite.linq4j.tree.Expression;
import org.apache.calcite.linq4j.tree.ExpressionType;
import org.apache.calcite.linq4j.tree.Expressions;
import org.apache.calcite.linq4j.tree.Primitive;
import org.apache.calcite.runtime.SqlFunctions;
import org.apache.ignite.internal.processors.query.calcite.util.IgniteMath;
import org.jetbrains.annotations.Nullable;

/** Calcite liq4j expressions customized for Ignite. */
public class IgniteExpressions {
/** Make binary expression with arithmetic operations override. */
public static Expression makeBinary(ExpressionType binaryType, Expression left, Expression right) {
switch (binaryType) {
case Add:
return addExact(left, right);
case Subtract:
return subtractExact(left, right);
case Multiply:
return multiplyExact(left, right);
case Divide:
return divideExact(left, right);
default:
return Expressions.makeBinary(binaryType, left, right);
}
}

/** Make unary expression with arithmetic operations override. */
public static Expression makeUnary(ExpressionType unaryType, Expression operand) {
switch (unaryType) {
case Negate:
case NegateChecked:
return negateExact(unaryType, operand);
default:
return Expressions.makeUnary(unaryType, operand);
}
}

/** Generate expression for method IgniteMath.addExact() for integer subtypes. */
public static Expression addExact(Expression left, Expression right) {
if (larger(left.getType(), right.getType()).isFixedNumeric())
return Expressions.call(IgniteMath.class, "addExact", left, right);

return Expressions.makeBinary(ExpressionType.Add, left, right);
}

/** Generate expression for method IgniteMath.subtractExact() for integer subtypes. */
public static Expression subtractExact(Expression left, Expression right) {
if (larger(left.getType(), right.getType()).isFixedNumeric())
return Expressions.call(IgniteMath.class, "subtractExact", left, right);

return Expressions.makeBinary(ExpressionType.Subtract, left, right);
}

/** Generate expression for method IgniteMath.multiplyExact() for integer subtypes. */
public static Expression multiplyExact(Expression left, Expression right) {
if (larger(left.getType(), right.getType()).isFixedNumeric())
return Expressions.call(IgniteMath.class, "multiplyExact", left, right);

return Expressions.makeBinary(ExpressionType.Multiply, left, right);
}

/** Generate expression for method IgniteMath.divideExact() for integer subtypes. */
public static Expression divideExact(Expression left, Expression right) {
if (larger(left.getType(), right.getType()).isFixedNumeric())
return Expressions.call(IgniteMath.class, "divideExact", left, right);

return Expressions.makeBinary(ExpressionType.Divide, left, right);
}

/** Generate expression for method IgniteMath.negateExact() for integer subtypes. */
private static Expression negateExact(ExpressionType unaryType, Expression operand) {
assert unaryType == ExpressionType.Negate || unaryType == ExpressionType.NegateChecked;

Type opType = operand.getType();

if (opType == Integer.TYPE || opType == Long.TYPE || opType == Short.TYPE || opType == Byte.TYPE)
return Expressions.call(IgniteMath.class, "negateExact", operand);

return Expressions.makeUnary(unaryType, operand);
}

/** Generate expression for conversion from numeric primitive to numeric primitive with bounds check. */
public static Expression convertChecked(Expression exp, Primitive fromPrimitive, Primitive toPrimitive) {
if (fromPrimitive.ordinal() <= toPrimitive.ordinal() || !toPrimitive.isFixedNumeric())
return Expressions.convert_(exp, toPrimitive.primitiveClass);

return Expressions.call(IgniteMath.class, "convertTo" +
SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", exp);
}

/** Generate expression for conversion from string to numeric primitive with bounds check. */
public static Expression parseStringChecked(Expression exp, Primitive toPrimitive) {
return Expressions.call(IgniteMath.class, "convertTo" +
SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", Expressions.new_(BigDecimal.class, exp));
}

/** Generate expression for conversion from Number to numeric primitive with bounds check. */
public static Expression unboxChecked(Expression exp, @Nullable Primitive fromBox, Primitive toPrimitive) {
if ((fromBox != null && fromBox.ordinal() <= toPrimitive.ordinal()) || !toPrimitive.isFixedNumeric())
return Expressions.unbox(exp, toPrimitive);

return Expressions.call(IgniteMath.class, "convertTo" +
SqlFunctions.initcap(toPrimitive.primitiveName) + "Exact", exp);
}

/** Find larger in type hierarchy. */
private static Primitive larger(Type type0, Type type1) {
Primitive primitive0 = Primitive.ofBoxOr(type0);
Primitive primitive1 = Primitive.ofBoxOr(type1);

return primitive0.ordinal() > primitive1.ordinal() ? primitive0 : primitive1;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.calcite.util.TypeUtils;
import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -36,8 +37,21 @@ public IgniteRexBuilder(RelDataTypeFactory typeFactory) {

/** {@inheritDoc} */
@Override protected RexLiteral makeLiteral(@Nullable Comparable o, RelDataType type, SqlTypeName typeName) {
if (o != null && typeName == SqlTypeName.DECIMAL && TypeUtils.hasScale(type))
return super.makeLiteral(((BigDecimal)o).setScale(type.getScale(), RoundingMode.HALF_UP), type, typeName);
if (o != null && typeName == SqlTypeName.DECIMAL) {
BigDecimal bd = (BigDecimal)o;

if (type.getSqlTypeName() == SqlTypeName.BIGINT) {
try {
bd.longValueExact();
}
catch (ArithmeticException e) {
throw new IgniteSQLException(SqlTypeName.BIGINT.getName() + " overflow", e);
}
}

if (TypeUtils.hasScale(type))
return super.makeLiteral(bd.setScale(type.getScale(), RoundingMode.HALF_UP), type, typeName);
}

return super.makeLiteral(o, type, typeName);
}
Expand Down
Loading

0 comments on commit d8e8e09

Please sign in to comment.